return invalid surfaceindex (index is 0 based)

Don't check for intersecting mapped trigs if only volume elements are
added (and no mapped surface elements).

In case of only inserted volume elements, we don't care about the limit
at special points.

.clang-format

@@ -0,0 +1,64 @@
+Language: Cpp
+BasedOnStyle: LLVM
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands: true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: InlineOnly
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterClass: true
+  AfterControlStatement: true
+  AfterEnum: true
+  AfterFunction: true
+  AfterNamespace: true
+  AfterStruct: true
+  AfterUnion: true
+  BeforeCatch: true
+  BeforeElse: true
+  IndentBraces: true
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakInheritanceList: AfterColon
+ColumnLimit: 0
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 2
+ContinuationIndentWidth: 2
+Cpp11BracedListStyle: true
+EmptyLineBeforeAccessModifier: Never
+IndentCaseLabels: false
+IndentPPDirectives: None
+IndentWidth: 2
+KeepEmptyLinesAtTheStartOfBlocks: false
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+PointerAlignment: Left
+ReflowComments: true
+SortIncludes: false
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceBeforeParens: Custom
+SpaceBeforeParensOptions:
+  AfterControlStatements: true
+  AfterFunctionDefinitionName: true
+  AfterFunctionDeclarationName: true
+SpacesInAngles: false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInSquareBrackets: false
+Standard: Latest
+TabWidth: 2
+UseTab: Never
+

.gitignore

@@ -0,0 +1,11 @@
+*.whl
+dist
+build
+*.vol.gz
+*.vol
+*.ini
+__pycache__
+*.json
+*.zip
+.cache
+*.patch

.gitlab-ci.yml

@@ -32,13 +32,23 @@ push_github:
     - "echo off"
     - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64"
     - set CI_DIR=C:\ci\%CI_PIPELINE_ID%
-    - set CLCACHE_BASEDIR=C:\ci\%CI_PIPELINE_ID%
+    - set CCACHE_BASEDIR=C:\ci\%CI_PIPELINE_ID%
     - set NETGEN_BUILD_DIR=%CI_DIR%\build
     - set INSTALL_DIR=%CI_DIR%\install
     - set SRC_DIR=%CI_DIR%\src
     - set NETGENDIR=%INSTALL_DIR%\bin
     - set PYTHONPATH=%INSTALL_DIR%\lib\site-packages
-    - set PATH=%NETGENDIR%;%PATH%
+    - echo %PATH%
+    - set PATH=%INSTALL_DIR%\bin;C:\python312;C:\python312\bin;C:\python312\Scripts;C:\tools\;%PATH%
+    - echo %PATH%
+    - set CCACHE_HARDLINK=1
+    - set CCACHE_NOHASHDIR=1
+    - C:\tools\ccache -s
+    - C:\tools\ccache -M 20G
+    - dir C:\python312
+    - python.exe --version
+    - python.exe -m pip install -U netgen-occt netgen-occt-devel
+    - cmake --version
 
 build_win:
   <<: *win
@@ -54,12 +64,14 @@ build_win:
     - >-
         cmake %SRC_DIR%
         -G Ninja
+        -DCMAKE_PREFIX=C:/python312
+        -DPython3_ROOT_DIR=C:/python312
         -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR%
         -DCHECK_RANGE=ON
         -DUSE_CGNS=ON
         -DUSE_OCC=ON
         -DUSE_CCACHE=ON
-        -DENABLE_UNIT_TESTS=ON
+        -DENABLE_UNIT_TESTS=OFF
         -DCMAKE_BUILD_TYPE=Release
     - cmake --build . --target install --config Release
 
@@ -74,6 +86,21 @@ test_win:
     - cd ..
   needs: ["build_win"]
 
+generate_results:
+  <<: *win
+  stage: test
+  script:
+    - pip install pytest-check
+    - cd tests\pytest
+    - python test_tutorials.py new_results.json
+  needs: ["build_win"]
+  when: manual
+  artifacts:
+    paths:
+      - tests/pytest/new_results.json
+    when: always
+    expire_in: 1 week
+
 cleanup_win:
   <<: *win
   stage: cleanup
@@ -177,15 +204,6 @@ test_build_ngsolve:
       netgen_${CI_PIPELINE_ID}_installed:${UBUNTU_VERSION}
       bash -c 'cd /root/src/netgen/tests/ && ./build_ngsolve.sh'
 
-# cpp guideline checks
-# test_guidelines:
-#   <<: *ubuntu
-#   stage: test
-#   script:
-#     - docker run -e CCACHE_DIR=/ccache -v /mnt/ccache:/ccache netgen_${CI_PIPELINE_ID}:${UBUNTU_VERSION} bash /root/src/netgen/tests/build_guidelines.sh
-#   when: always
-#   allow_failure: true
-
 cleanup_ubuntu:
   stage: cleanup
   tags:
@@ -263,7 +281,7 @@ cleanup_mac:
   needs: ["test_mac"]
 
 pip_linux:
-  image: quay.io/pypa/manylinux2014_x86_64
+  image: quay.io/pypa/manylinux_2_28_x86_64
   stage: build
   tags:
     - pip
@@ -279,11 +297,11 @@ pip_windows:
     - pip
     - windows
   script:
-    - .\tests\build_pip.ps1 C:\Python38
+    - .\tests\build_pip.ps1 C:\Python313
-    - .\tests\build_pip.ps1 C:\Python39
-    - .\tests\build_pip.ps1 C:\Python310
-    - .\tests\build_pip.ps1 C:\Python311
     - .\tests\build_pip.ps1 C:\Python312
+    - .\tests\build_pip.ps1 C:\Python311
+    - .\tests\build_pip.ps1 C:\Python310
+    - .\tests\build_pip.ps1 C:\Python39
   when: manual
 
 pip_macos:
@@ -293,9 +311,9 @@ pip_macos:
     - macosx
     - m1
   script:
-    - ./tests/build_pip_mac.sh 3.8
+    - ./tests/build_pip_mac.sh 3.13
-    - ./tests/build_pip_mac.sh 3.9
-    - ./tests/build_pip_mac.sh 3.10
-    - ./tests/build_pip_mac.sh 3.11
     - ./tests/build_pip_mac.sh 3.12
+    - ./tests/build_pip_mac.sh 3.11
+    - ./tests/build_pip_mac.sh 3.10
+    - ./tests/build_pip_mac.sh 3.9
   when: manual

CMakeLists.txt

@@ -16,7 +16,7 @@ option( USE_GUI     "build with GUI" ON )
 option( USE_PYTHON  "build with python interface" ON )
 cmake_dependent_option( PREFER_SYSTEM_PYBIND11  "Use system wide PyBind11" OFF "USE_PYTHON" OFF)
 option( USE_MPI     "enable mpi parallelization" OFF )
-option( USE_MPI4PY  "enable mpi4py interface" ON )
+option( USE_MPI_WRAPPER "enable mpi wrapper (run-time dispatch of MPI library calls)" OFF )
 option( USE_OCC     "build with OpenCascade geometry kernel interface" ON)
 option( USE_STLGEOM     "build with STL geometry support" ON)
 option( USE_CSG     "build with CSG kernel" ON)
@@ -87,12 +87,15 @@ set(NG_INSTALL_SUFFIX netgen CACHE STRING "Suffix appended to install directorie
 if(USE_PYTHON)
   if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.18)
       find_package(Python3 REQUIRED COMPONENTS Development.Module)
+      if(NOT EMSCRIPTEN)
           find_package(Python3 COMPONENTS Interpreter Development.Embed)
+      endif()
   else()
     find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
   endif()
 
   if(NOT CMAKE_CROSSCOMPILING)
+      find_package(Python3 REQUIRED COMPONENTS Interpreter)
       execute_process(COMMAND ${Python3_EXECUTABLE} -c "import os.path, sysconfig;print(os.path.relpath(sysconfig.get_path('platlib'), sysconfig.get_path('data')))" OUTPUT_VARIABLE PYTHON_PACKAGES_INSTALL_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
       file(TO_CMAKE_PATH ${PYTHON_PACKAGES_INSTALL_DIR} PYTHON_PACKAGES_INSTALL_DIR)
   endif(NOT CMAKE_CROSSCOMPILING)
@@ -161,6 +164,7 @@ if(USE_CCACHE)
   find_program(CCACHE_FOUND NAMES ccache ccache.bat)
   if(CCACHE_FOUND)
     set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+    message(STATUS "Using ccache ${CCACHE_FOUND}")
   endif(CCACHE_FOUND)
 endif(USE_CCACHE)
 
@@ -319,6 +323,7 @@ if (USE_PYTHON)
       add_subdirectory(external_dependencies/pybind11)
     endif()
 
+    target_compile_definitions(netgen_python INTERFACE NG_PYTHON NETGEN_PYTHON)
     target_include_directories(netgen_python INTERFACE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS})
     target_include_directories(nglib PRIVATE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS})
     if(Python3_LIBRARIES AND (WIN32 OR NOT BUILD_FOR_CONDA))
@@ -332,28 +337,16 @@ if (USE_PYTHON)
 endif (USE_PYTHON)
 
 #######################################################################
-add_library(netgen_mpi INTERFACE)
 add_library(netgen_metis INTERFACE)
 if (USE_MPI)
-    find_package(MPI REQUIRED)
+    set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
-    target_include_directories(netgen_mpi INTERFACE ${MPI_CXX_INCLUDE_PATH})
+    find_package(MPI)
-    target_link_libraries(netgen_mpi INTERFACE ${MPI_mpi_LIBRARY} ${MPI_CXX_LIBRARIES} )
-    target_compile_definitions(netgen_mpi INTERFACE PARALLEL )
 
     find_package(METIS REQUIRED)
     target_include_directories(netgen_metis INTERFACE ${METIS_INCLUDE_DIR})
     target_link_libraries(netgen_metis INTERFACE ${METIS_LIBRARY} )
     target_compile_definitions(netgen_metis INTERFACE METIS )
-
-    if(USE_MPI4PY AND USE_PYTHON)
-      execute_process(COMMAND ${Python3_EXECUTABLE} -c "import mpi4py;print(mpi4py.get_include())" OUTPUT_VARIABLE mpi4py_path OUTPUT_STRIP_TRAILING_WHITESPACE)
-      find_path(MPI4PY_INCLUDE_DIR mpi4py.h HINTS ${mpi4py_path}/mpi4py NO_DEFAULT_PATH REQUIRED)
-      target_include_directories(netgen_metis INTERFACE ${MPI4PY_INCLUDE_DIR})
-      target_compile_definitions(netgen_metis INTERFACE NG_MPI4PY )
-      message(STATUS "Found mpi4py: ${MPI4PY_INCLUDE_DIR}")
-    endif(USE_MPI4PY AND USE_PYTHON)
 endif (USE_MPI)
-install(TARGETS netgen_mpi netgen_metis ${NG_INSTALL_DIR})
 
 #######################################################################
 add_library(occ_libs INTERFACE IMPORTED)
@@ -372,35 +365,46 @@ if (USE_OCC)
       TKGeomAlgo
       TKGeomBase
       TKHLR
-      TKIGES
       TKLCAF
       TKMath
       TKMesh
       TKOffset
       TKPrim
-      TKSTEP
-      TKSTEP209
-      TKSTEPAttr
-      TKSTEPBase
-      TKSTL
       TKService
       TKShHealing
       TKTopAlgo
       TKV3d
       TKVCAF
       TKXCAF
-      TKXDEIGES
-      TKXDESTEP
       TKXSBase
       TKernel
     )
+    if(${OpenCASCADE_MAJOR_VERSION}.${OpenCASCADE_MINOR_VERSION} VERSION_GREATER_EQUAL 7.8)
+      list(APPEND OCC_LIBRARIES TKDEIGES TKDESTEP TKDESTL)
+    else()
+      list(APPEND OCC_LIBRARIES
+        TKIGES
+        TKSTEP
+        TKSTL
+        TKXDEIGES
+        TKXDESTEP
+        TKSTEP209
+        TKSTEPAttr
+        TKSTEPBase
+        )
+    endif()
     if(UNIX AND NOT APPLE)
       list(PREPEND OCC_LIBRARIES -Wl,--start-group)
       list(APPEND OCC_LIBRARIES -Wl,--end-group)
     endif()
 
     target_link_libraries(occ_libs INTERFACE ${OCC_LIBRARIES})
-    include_directories(${OpenCASCADE_INCLUDE_DIR})
+    get_target_property(occ_include_dir TKernel INTERFACE_INCLUDE_DIRECTORIES)
+    if(NOT occ_include_dir)
+      set(occ_include_dir ${OpenCASCADE_INCLUDE_DIR})
+    endif()
+    target_include_directories(occ_libs INTERFACE ${occ_include_dir})
+    message(STATUS "OpenCasCade include dirs: ${occ_include_dir}")
     if(NOT OpenCASCADE_BUILD_SHARED_LIBS)
       if(OpenCASCADE_WITH_FREETYPE)
         find_library( FREETYPE NAMES freetype HINTS ${OpenCASCADE_LIBRARY_DIR})
@@ -415,10 +419,12 @@ if (USE_OCC)
       find_package(Threads REQUIRED)
       target_link_libraries(occ_libs INTERFACE Threads::Threads)
     endif()
-    message(STATUS "OCC DIRS ${OpenCASCADE_INCLUDE_DIR}")
     if(WIN32 AND USE_GUI)
-      target_link_libraries(nggui PRIVATE occ_libs Ws2_32.lib)
+      target_link_libraries(nggui PRIVATE Ws2_32.lib)
     endif(WIN32 AND USE_GUI)
+    if(USE_GUI)
+      target_link_libraries(nggui PRIVATE occ_libs)
+    endif(USE_GUI)
 endif (USE_OCC)
 
 #######################################################################
@@ -627,6 +633,16 @@ if(UNIX)
     endif(temp)
 endif(UNIX)
 
+if(USE_PYTHON AND NOT SKBUILD)
+  # install egg file to let python/pip know that Netgen ist installed
+  file( WRITE "netgen_mesher-py3.egg-info"
+"Metadata-Version: 2.1
+Name: netgen-mesher
+Version: ${NETGEN_VERSION_MAJOR}.${NETGEN_VERSION_MINOR}.${NETGEN_VERSION_PATCH}.post${NETGEN_VERSION_TWEAK}
+")
+  install(FILES netgen_mesher-py3.egg-info DESTINATION ${NG_INSTALL_DIR_PYTHON} COMPONENT netgen)
+endif()
+
 if(APPLE AND NOT SKBUILD)
     # create some auxiliary files
     set(mac_startup ${CMAKE_CURRENT_BINARY_DIR}/startup.sh)

README.md

@@ -4,3 +4,4 @@ NETGEN is an automatic 3d tetrahedral mesh generator. It accepts input from cons
 
 Find the Open Source Community on https://ngsolve.org
 Support & Services: https://cerbsim.com
+

cmake/SuperBuild.cmake

@@ -57,7 +57,7 @@ set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_C_COMPILER)
 set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_CXX_COMPILER)
 set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_BUILD_TYPE)
 
-set(SUBPROJECT_CMAKE_ARGS "${SUBPROJECT_CMAKE_ARGS};-DCMAKE_POSITION_INDEPENDENT_CODE=ON" CACHE INTERNAL "")
+set(SUBPROJECT_CMAKE_ARGS "${SUBPROJECT_CMAKE_ARGS};-DCMAKE_POSITION_INDEPENDENT_CODE=ON;-DCMAKE_POLICY_VERSION_MINIMUM=3.5" CACHE INTERNAL "")
 
 if(USE_CCACHE)
   find_program(CCACHE_FOUND NAMES ccache ccache.bat)
@@ -89,8 +89,12 @@ if(BUILD_OCC)
   set(OCC_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/occ)
 
   ExternalProject_Add(project_occ
-    URL https://github.com/Open-Cascade-SAS/OCCT/archive/refs/tags/V7_6_3.zip
+    # URL https://github.com/Open-Cascade-SAS/OCCT/archive/refs/tags/V7_6_3.zip
-    URL_MD5 2426e373903faabbd4f96a01a934b66d
+    # URL_MD5 2426e373903faabbd4f96a01a934b66d
+    # URL https://github.com/Open-Cascade-SAS/OCCT/archive/refs/tags/V7_7_2.zip
+    # URL_MD5 533eb4f18af0f77ae321b158caeaee79
+    URL https://github.com/Open-Cascade-SAS/OCCT/archive/refs/tags/V7_8_1.zip
+    URL_MD5 bf62952a03696dab9e4272aa8efacb1a
     DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies
     ${SUBPROJECT_ARGS}
     CMAKE_ARGS
@@ -116,28 +120,15 @@ if(BUILD_OCC)
   list(APPEND NETGEN_DEPENDENCIES project_occ)
   set(OpenCascade_ROOT ${OCC_DIR})
 else(BUILD_OCC)
-    if(WIN32 AND NOT OCC_INCLUDE_DIR AND NOT OpenCASCADE_DIR)
-        # we can download prebuilt occ binaries for windows
-        ExternalProject_Add(win_download_occ
-          ${SUBPROJECT_ARGS}
-          URL ${OCC_DOWNLOAD_URL_WIN}
-          UPDATE_COMMAND "" # Disable update
-          BUILD_IN_SOURCE 1
-          CONFIGURE_COMMAND ""
-          BUILD_COMMAND ""
-          INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory . ${CMAKE_INSTALL_PREFIX}
-          )
-        list(APPEND NETGEN_DEPENDENCIES win_download_occ)
-    else()
     find_package(OpenCascade NAMES OpenCasCade OpenCASCADE opencascade)
     if(NOT OpenCascade_FOUND)
       message(FATAL_ERROR "Opencascade not found, either\n\
+      - install pip packages netgen-occt-devel netgen-occ\n\
       - set OpenCascade_DIR to a directory containting opencascadeConfig.cmake\n\
       - build OpenCascade automatically by passing -DBUILD_OCC=ON\n\
       - disable OpenCascade by passing -DUSE_OCC=OFF\n\
       ")
     endif()
-    endif()
 endif(BUILD_OCC)
 endif(USE_OCC)
 
@@ -160,9 +151,11 @@ if(BUILD_ZLIB)
     # force linking the static library
     set(ZLIB_INCLUDE_DIRS ${ZLIB_ROOT}/include)
     set(ZLIB_LIBRARIES ${ZLIB_ROOT}/lib/zlibstatic.lib)
-  elseif(EMSCRIPTEN)
+    set(ZLIB_LIBRARY_RELEASE ${ZLIB_ROOT}/lib/zlibstatic.lib)
+  else(WIN32)
     set(ZLIB_INCLUDE_DIRS ${ZLIB_ROOT}/include)
     set(ZLIB_LIBRARIES ${ZLIB_ROOT}/lib/libz.a)
+    set(ZLIB_LIBRARY_RELEASE ${ZLIB_ROOT}/lib/libz.a)
   endif(WIN32)
 else()
     include(cmake/external_projects/zlib.cmake)
@@ -184,7 +177,9 @@ if (USE_PYTHON)
     endif( PYBIND_INCLUDE_DIR )
     if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.18)
       find_package(Python3 COMPONENTS Interpreter Development.Module)
+      if(NOT EMSCRIPTEN)
           find_package(Python3 COMPONENTS Interpreter Development.Embed)
+      endif()
     else()
       find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
     endif()
@@ -211,7 +206,6 @@ endif(USE_CGNS)
 
 #######################################################################
 if(USE_MPI)
-  if(UNIX)
     if (METIS_DIR)
       message(STATUS "Using external METIS at: ${METIS_DIR}")
     else (METIS_DIR)
@@ -222,9 +216,6 @@ if(USE_MPI)
 	include(cmake/external_projects/metis.cmake)
       endif(NOT METIS_FOUND)
     endif(METIS_DIR)
-  else(UNIX)
-    find_package(METIS REQUIRED)
-  endif(UNIX)
 endif(USE_MPI)
 
 
@@ -242,6 +233,7 @@ set_vars( NETGEN_CMAKE_ARGS
   USE_GUI
   USE_PYTHON
   USE_MPI
+  USE_MPI_WRAPPER
   USE_VT
   USE_VTUNE
   USE_NUMA
@@ -267,6 +259,7 @@ set_vars( NETGEN_CMAKE_ARGS
   OpenCascade_ROOT
   ZLIB_INCLUDE_DIRS
   ZLIB_LIBRARIES
+  ZLIB_LIBRARY_RELEASE
   ZLIB_ROOT
 
   NGLIB_LIBRARY_TYPE

cmake/external_projects/metis.cmake

@@ -3,8 +3,8 @@ set(METIS_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/metis)
 
 ExternalProject_Add(project_metis
   PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dependencies
-  URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p6.tar.gz
+  URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p12.tar.gz
-  URL_MD5 55fc654bb838846b856ba898795143f1
+  URL_MD5 6cd66f75f88dfa2cf043de011f85d8bc
   DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies
   CMAKE_ARGS
          -DGKLIB_PATH=${METIS_SRC_DIR}/GKlib

cmake/external_projects/tcltk.cmake

@@ -51,12 +51,17 @@ if(APPLE OR WIN32)
         NO_SYSTEM_ENVIRONMENT_PATH
         NO_CMAKE_SYSTEM_PATH
         NO_CMAKE_FIND_ROOT_PATH
-        HINTS ${PYTHON_PREFIX}/lib ${PYTHON_PREFIX}/tcl
+        HINTS
+        ${PYTHON_PREFIX}/lib
+        ${PYTHON_PREFIX}/tcl
+        ${PYTHON_PREFIX}/Frameworks
+        ${PYTHON_PREFIX}/Frameworks/Tcl.framework
+        ${PYTHON_PREFIX}/Frameworks/Tk.framework
         )
     find_library(TCL_STUB_LIBRARY NAMES tclstub85 tclstub8.5 tclstub86 tclstub8.6 ${tcl_find_args})
     find_library(TK_STUB_LIBRARY NAMES tkstub85 tkstub8.5 tkstub86 tkstub8.6 ${tcl_find_args})
-    find_library(TCL_LIBRARY NAMES tcl85 tcl8.5 tcl86 tcl8.6 tcl86t ${tcl_find_args})
+    find_library(TCL_LIBRARY NAMES tcl85 tcl8.5 tcl86 tcl8.6 tcl86t Tcl ${tcl_find_args})
-    find_library(TK_LIBRARY NAMES tk85 tk8.5 tk86 tk8.6 tk86t ${tcl_find_args})
+    find_library(TK_LIBRARY NAMES tk85 tk8.5 tk86 tk8.6 tk86t Tk ${tcl_find_args})
 else()
     # use system tcl/tk on linux
     find_package(TclStub REQUIRED)
@@ -104,6 +109,7 @@ if(APPLE)
            -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/Contents/MacOS
 	   -DTCL_INCLUDE_PATH=${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tcl.framework/Headers
 	   -DTK_INCLUDE_PATH=${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tk.framework/Headers
+     -DCMAKE_POLICY_VERSION_MINIMUM=3.5
     ${SUBPROJECT_ARGS}
   )
 

cmake/generate_version_file.cmake

@@ -106,6 +106,7 @@ file(GENERATE OUTPUT netgen_config.hpp CONTENT
 #define NETGEN_USE_CHECK_RANGE          $<BOOL:${CHECK_RANGE}>
 #define NETGEN_BUILD_STUB_FILES         $<BOOL:${BUILD_STUB_FILES}>
 #define NETGEN_BUILD_FOR_CONDA          $<BOOL:${BUILD_FOR_CONDA}>
+#define NETGEN_SHARED_LIBRARY_SUFFIX    \"${CMAKE_SHARED_LIBRARY_SUFFIX}\"
 
 #endif // NETGEN_CONFIG_HPP_INCLUDED___
 ")

external_dependencies/pybind11

@@ -1 +1 @@
-Subproject commit 80dc998efced8ceb2be59756668a7e90e8bef917
+Subproject commit 38bf7b174875c27c1ba98bdf5a9bf13d967f14d4

libsrc/core/.clang-tidy

@@ -1,4 +1,4 @@
-Checks: '*,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers'
+Checks: '*,-cppcoreguidelines-avoid-non-const-global-variables,-llvmlibc-restrict-system-libc-headers,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers'
 CheckOptions:
         - key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
           value: 1

libsrc/core/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library(ngcore ${NGCORE_LIBRARY_TYPE}
   taskmanager.cpp
   utils.cpp
   version.cpp
+  ng_mpi_wrapper.cpp
   )
 
 string(REPLACE "|" ";" ng_compile_flags_replace_sep "${NG_COMPILE_FLAGS}")
@@ -23,8 +24,30 @@ if(EMSCRIPTEN)
   target_compile_options(ngcore PUBLIC -sNO_DISABLE_EXCEPTION_CATCHING)
 endif()
 
-if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
+if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND USE_PYTHON)
-    target_link_libraries(ngcore PUBLIC stdc++fs)
+  # Python packages on Linux are compiled with the old ABI,
+  # make sure that the same ABI is used in plugins aswell
+  try_run(
+    ret_val can_compile
+    ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/_get_glibcxx_use_cxx11_abi.cpp
+    RUN_OUTPUT_VARIABLE use_glibcxx_cxx11_abi
+    )
+  target_compile_definitions(ngcore PUBLIC -D_GLIBCXX_USE_CXX11_ABI=${use_glibcxx_cxx11_abi})
+  try_run(
+    ret_val can_compile
+    ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/_get_gxx_abi.cpp
+    RUN_OUTPUT_VARIABLE default_cxx_abi_version
+    )
+  if(${can_compile} AND (${ret_val} EQUAL 0))
+    # Different python modules using pybind11 need to use the same C++ ABI version
+    # for compatibility
+    set(cxx_abi_version 17)
+    if(cxx_abi_version LESS default_cxx_abi_version)
+      set(cxx_abi_version ${default_cxx_abi_version})
+    endif()
+    message(STATUS "GNU C++ ABI version: ${cxx_abi_version}")
+    target_compile_options(ngcore PUBLIC "-fabi-version=${cxx_abi_version}")
+  endif()
 endif()
 
 if(USE_PYTHON)
@@ -33,10 +56,12 @@ if(USE_PYTHON)
 endif(USE_PYTHON)
 
 if(WIN32)
-  target_compile_options(ngcore PUBLIC /bigobj /MP /W1 /wd4068)
+  target_compile_options(ngcore PUBLIC /bigobj $<BUILD_INTERFACE:/MP;/W1;/wd4068>)
   get_WIN32_WINNT(ver)
   target_compile_definitions(ngcore PUBLIC _WIN32_WINNT=${ver} WNT WNT_WINDOW NOMINMAX MSVC_EXPRESS _CRT_SECURE_NO_WARNINGS HAVE_STRUCT_TIMESPEC WIN32)
   target_link_options(ngcore PUBLIC /ignore:4273 /ignore:4217 /ignore:4049)
+else(WIN32)
+  target_link_libraries(ngcore PUBLIC dl)
 endif(WIN32)
 
 target_compile_definitions(ngcore PRIVATE NGCORE_EXPORTS)
@@ -62,7 +87,7 @@ endif(USE_NUMA)
 
 install(TARGETS ngcore DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
 
-target_link_libraries(ngcore PUBLIC netgen_mpi PRIVATE "$<BUILD_INTERFACE:netgen_python>" ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(ngcore PRIVATE "$<BUILD_INTERFACE:netgen_python>" ${CMAKE_THREAD_LIBS_INIT})
 
 install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp memtracer.hpp
   exception.hpp symboltable.hpp paje_trace.hpp utils.hpp profiler.hpp mpi_wrapper.hpp
@@ -70,6 +95,7 @@ install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp
   xbool.hpp signal.hpp bitarray.hpp table.hpp hashtable.hpp ranges.hpp ngstream.hpp
   simd.hpp simd_avx.hpp simd_avx512.hpp simd_generic.hpp simd_sse.hpp simd_arm64.hpp
   register_archive.hpp autodiff.hpp autodiffdiff.hpp
+  ng_mpi.hpp ng_mpi_generated_declarations.hpp mpi4py_pycapi.h ng_mpi_native.hpp
   DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel)
 
 if(ENABLE_CPP_CORE_GUIDELINES_CHECK)
@@ -80,7 +106,7 @@ add_dependencies(ngcore ng_generate_version_file)
 
 if(USE_PYTHON)
   pybind11_add_module(pyngcore MODULE python_ngcore_export.cpp)
-  target_link_libraries(pyngcore PUBLIC ngcore netgen_python)
+  target_link_libraries(pyngcore PUBLIC ngcore PRIVATE netgen_python)
   set_target_properties(pyngcore PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/../${NETGEN_PYTHON_RPATH}")
   if(EMSCRIPTEN)
     target_compile_definitions(pyngcore PRIVATE NGCORE_EXPORTS)
@@ -88,3 +114,62 @@ if(USE_PYTHON)
   install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON}/pyngcore COMPONENT netgen)
 endif(USE_PYTHON)
 
+function (build_mpi_variant)
+  set(target ng_${ARGV0})
+  set(include_dir ${ARGV1})
+  message("1Building MPI variant: ${ARGV0} ${ARGV1}")
+  add_library(${target} SHARED ng_mpi.cpp)
+  target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python)
+  target_compile_definitions(${target} PUBLIC PARALLEL NG_MPI_WRAPPER)
+  target_include_directories(${target} PRIVATE ${include_dir})
+  set_target_properties(${target} PROPERTIES PREFIX "")
+  install(TARGETS ${target} RUNTIME DESTINATION ${NG_INSTALL_DIR_BIN} LIBRARY DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen)
+endfunction()
+
+if(USE_MPI)
+  target_compile_definitions(ngcore PUBLIC PARALLEL)
+
+  message(STATUS "Found MPI version\n${MPI_C_LIBRARY_VERSION_STRING}")
+
+  if(USE_MPI_WRAPPER)
+    target_compile_definitions(ngcore PUBLIC NG_MPI_WRAPPER)
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Microsoft MPI.*")
+      set(MICROSOFT_MPI_INCLUDE_DIR ${MPI_C_HEADER_DIR})
+      set(MICROSOFT_MPI_LIBRARY ${MPI_msmpi_LIBRARY})
+    endif()
+
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*")
+      set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
+    endif()
+
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "MPICH.*")
+      set(MPICH_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
+    endif()
+
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Intel.*")
+      set(INTEL_MPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
+    endif()
+
+    if(OPENMPI_INCLUDE_DIR)
+      build_mpi_variant(openmpi ${OPENMPI_INCLUDE_DIR})
+    endif()
+    if(MPICH_INCLUDE_DIR)
+      build_mpi_variant(mpich ${MPICH_INCLUDE_DIR})
+    endif()
+    if(INTEL_MPI_INCLUDE_DIR)
+      build_mpi_variant(intel_mpi ${INTEL_MPI_INCLUDE_DIR})
+      if(WIN32)
+        target_link_libraries(ng_intel_mpi PUBLIC ${INTEL_MPI_LIBRARY})
+      endif()
+    endif()
+    if(MICROSOFT_MPI_INCLUDE_DIR)
+      build_mpi_variant(microsoft_mpi ${MICROSOFT_MPI_INCLUDE_DIR})
+      target_link_libraries(ng_microsoft_mpi PUBLIC ${MICROSOFT_MPI_LIBRARY})
+    endif()
+  else()
+    target_link_libraries(ngcore PUBLIC ${MPI_C_LIBRARIES})
+    target_include_directories(ngcore PUBLIC ${MPI_C_INCLUDE_PATH})
+  endif(USE_MPI_WRAPPER)
+
+endif(USE_MPI)
+

libsrc/core/_get_glibcxx_use_cxx11_abi.cpp

@@ -0,0 +1,13 @@
+#include <iostream>
+
+int main() {
+  #ifdef _GLIBCXX_USE_CXX11_ABI
+  if(_GLIBCXX_USE_CXX11_ABI)
+    std::cout << 1;
+  else
+    std::cout << 0;
+  #else // _GLIBCXX_USE_CXX11_ABI
+    std::cout << 0;
+  #endif // _GLIBCXX_USE_CXX11_ABI
+  return 0;
+}

libsrc/core/_get_gxx_abi.cpp

@@ -0,0 +1,7 @@
+#include <iostream>
+
+int main() {
+  if (__GXX_ABI_VERSION >= 2000 || __GXX_ABI_VERSION < 1000) return 1;
+  std::cout << (__GXX_ABI_VERSION % 100);
+  return 0;
+}

libsrc/core/archive.cpp

@@ -10,24 +10,31 @@
 namespace ngcore
 {
   // clang-tidy should ignore this static object
-  static std::unique_ptr<std::map<std::string, detail::ClassArchiveInfo>> type_register;  // NOLINT
+  // static std::map<std::string, detail::ClassArchiveInfo> type_register;  // NOLINT
+
+  auto& GetTypeRegister()
+  {
+    static std::map<std::string, detail::ClassArchiveInfo> type_register;
+    return type_register;
+  }
+  
   const detail::ClassArchiveInfo& Archive :: GetArchiveRegister(const std::string& classname)
   {
-    if(type_register == nullptr) type_register =
+    // if(type_register == nullptr) type_register =
-                                   std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
+    // std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
-    return (*type_register)[classname];
+    return GetTypeRegister()[classname];
   }
   void Archive :: SetArchiveRegister(const std::string& classname, const detail::ClassArchiveInfo& info)
   {
-    if(type_register == nullptr) type_register =
+    // if(type_register == nullptr) type_register =
-                                   std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
+    // std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
-    (*type_register)[classname] = info;
+    GetTypeRegister()[classname] = info;
   }
   bool Archive :: IsRegistered(const std::string& classname)
   {
-    if(type_register == nullptr) type_register =
+    // if(type_register == nullptr) type_register =
-                                   std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
+    // std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
-    return type_register->count(classname) != 0;
+    return GetTypeRegister().count(classname) != 0;
   }
 
 #ifdef NETGEN_PYTHON

libsrc/core/archive.hpp

@@ -1,6 +1,7 @@
 #ifndef NETGEN_CORE_ARCHIVE_HPP
 #define NETGEN_CORE_ARCHIVE_HPP
 
+#include <algorithm>
 #include <any>
 #include <array>                // for array
 #include <complex>              // for complex
@@ -42,6 +43,32 @@ namespace ngcore
     operator T&() { return val; }
   };
 
+  // Helper to detect shared_from_this
+  template <typename T>
+  class has_shared_from_this2
+    {
+    private:
+      // typedef T* T_ptr;
+      template <typename C> static std::true_type test(decltype(((C*)nullptr)->shared_from_this()));
+      template <typename C> static std::false_type test(...);
+      
+    public:
+      // If the test returns true_type, then T has shared_from_this
+      static constexpr bool value = decltype(test<T>(0))::value;
+  };
+  
+  
+
+  
+  template <typename T, typename = void>
+  class has_shallow_archive : public std::false_type {};
+  
+  template <typename T>
+  class has_shallow_archive<T, std::void_t<decltype(T::shallow_archive)>>
+    : public std::is_same<decltype(T::shallow_archive), std::true_type> {};
+  
+
+  
 #ifdef NETGEN_PYTHON
   pybind11::object CastAnyToPy(const std::any& a);
 #endif // NETGEN_PYTHON
@@ -51,7 +78,8 @@ namespace ngcore
   {
     template <class T, class Tuple, size_t... Is>
     T* construct_from_tuple(Tuple&& tuple, std::index_sequence<Is...> ) {
-      return new T{std::get<Is>(std::forward<Tuple>(tuple))...};
+      // return new T{std::get<Is>(std::forward<Tuple>(tuple))...};
+      return new T{std::get<Is>(std::move(tuple))...};
     }
 
     template <class T, class Tuple>
@@ -150,7 +178,8 @@ namespace ngcore
       void* (*downcaster)(const std::type_info&, void*);
 
       // Archive constructor arguments
-      std::function<void(Archive&, void*)> cargs_archiver;
+      // std::function<void(Archive&, void*)> cargs_archiver;
+      void (*cargs_archiver)(Archive&, void*);
 
 #ifdef NETGEN_PYTHON
       // std::function<pybind11::object(const std::any&)> anyToPyCaster;
@@ -485,6 +514,13 @@ namespace ngcore
     template <typename T>
     Archive& operator & (std::shared_ptr<T>& ptr)
     {
+      if constexpr(has_shallow_archive<T>::value)
+        if (shallow_to_python)
+          {
+            Shallow (ptr);
+            return *this;
+          }
+          
       if(Output())
         {
           // save -2 for nullptr
@@ -1101,15 +1137,32 @@ namespace ngcore
     { char c; *stream >> c; b = (c=='t'); return *this; }
     Archive & operator & (std::string & str) override
     {
+      // Ignore \r (carriage return) characters when reading strings
+      // this is necessary for instance when a file was written on Windows and is read on Unix
+
       int len;
       *stream >> len;
       char ch;
-      stream->get(ch); // '\n'
+      stream->get(ch); // read newline character
       if(ch == '\r') // windows line endings -> read \n as well
         stream->get(ch);
       str.resize(len);
       if(len)
         stream->get(&str[0], len+1, '\0');
+
+      // remove all \r characters from the string, check if size changed
+      // if so, read the remaining characters
+      str.erase(std::remove(str.begin(), str.end(), '\r'), str.cend());
+      size_t chars_to_read = len-str.size();
+      while (chars_to_read>0)
+      {
+        auto old_size = str.size();
+        str.resize(len);
+
+        stream->get(&str[old_size], chars_to_read+1, '\0');
+        str.erase(std::remove(str.begin()+old_size, str.end(), '\r'), str.cend());
+        chars_to_read = len - str.size();
+      }
       return *this;
     }
     Archive & operator & (char *& str) override

libsrc/core/array.hpp

@@ -216,6 +216,10 @@ namespace ngcore
   template <typename  T>
   constexpr T IndexBASE () { return T(0); }
 
+  template <typename  T>
+  constexpr T IndexBASE (T ind) { return IndexBASE<T>(); }
+
+  
 
   class IndexFromEnd
   {
@@ -278,7 +282,8 @@ namespace ngcore
     T first, next;
   public: 
     NETGEN_INLINE T_Range () { ; }
-    NETGEN_INLINE T_Range (T n) : first(0), next(n) {;}
+    // NETGEN_INLINE T_Range (T n) : first(0), next(n) {;}
+    NETGEN_INLINE explicit T_Range (size_t n) : first(IndexBASE<T>()), next(IndexBASE<T>()+n) {;}    
     NETGEN_INLINE T_Range (T f, T n) : first(f), next(n) {;}
     template <typename T2>
       NETGEN_INLINE T_Range(T_Range<T2> r2) : first(r2.First()), next(r2.Next()) { ; }
@@ -296,7 +301,7 @@ namespace ngcore
 
     NETGEN_INLINE T_Range Split (size_t nr, int tot) const
     {
-      T diff = next-first;
+      auto diff = next-first;
       return T_Range (first + nr * diff / tot,
                       first + (nr+1) * diff / tot);
     }
@@ -554,6 +559,13 @@ namespace ngcore
     // { return CArray<T> (data+pos); }
     NETGEN_INLINE T * operator+ (size_t pos) const { return data+pos; }
 
+    /// access first element. check by macro NETGEN_CHECK_RANGE
+    T & First () const
+    {
+      NETGEN_CHECK_RANGE(0,0,size);
+      return data[0];
+    }
+
     /// access last element. check by macro NETGEN_CHECK_RANGE
     T & Last () const
     {
@@ -687,6 +699,7 @@ namespace ngcore
     size_t allocsize;
     /// that's the data we have to delete, nullptr for not owning the memory
     T * mem_to_delete;
+    MemoryTracer mt;
 
 
     using FlatArray<T,IndexType>::size;
@@ -708,6 +721,7 @@ namespace ngcore
     {
       allocsize = asize; 
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*asize);
     }
 
 
@@ -717,7 +731,10 @@ namespace ngcore
     {
       allocsize = asize;
       if(ownMemory)
+      {
         mem_to_delete = adata;
+        mt.Alloc(sizeof(T)*asize);
+      }
       else
         mem_to_delete = nullptr;
     }
@@ -733,8 +750,7 @@ namespace ngcore
 
     NETGEN_INLINE Array (Array && a2) 
     {
-      mt.Swap(sizeof(T) * allocsize, a2.mt, sizeof(T) * a2.allocsize);
+      mt = std::move(a2.mt);
-
       size = a2.size; 
       data = a2.data;
       allocsize = a2.allocsize;
@@ -753,6 +769,7 @@ namespace ngcore
         {
           allocsize = size;
           mem_to_delete = data;
+          mt.Alloc(sizeof(T)*size);
           for (size_t i = 0; i < size; i++)
             data[i] = a2.data[i];
         }
@@ -772,6 +789,7 @@ namespace ngcore
     {
       allocsize = size;
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*size);
       /*
       for (size_t i = 0; i < size; i++)
         data[i] = a2[i];
@@ -788,6 +806,7 @@ namespace ngcore
     {
       allocsize = size;
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*size);
       size_t cnt = 0;
       for (auto val : list)
         data[cnt++] = val;
@@ -800,6 +819,7 @@ namespace ngcore
     {
       allocsize = size;
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*size);
       for(size_t i = 0; i <  a2.Size(); i++)
         data[i] = a2[i];
       for (size_t i = a2.Size(), j=0; i < size; i++,j++)
@@ -834,6 +854,9 @@ namespace ngcore
     NETGEN_INLINE void NothingToDelete () 
     { 
       mem_to_delete = nullptr;
+
+      // this memory is not managed by the Array anymore, so set the memory usage to 0
+      mt.Free(sizeof(T)*allocsize);
     }
 
     /// Change logical size. If necessary, do reallocation. Keeps contents.
@@ -947,7 +970,7 @@ namespace ngcore
 
 
     /// Delete element i. Move last element to position i.
-    NETGEN_INLINE void DeleteElement (size_t i)
+    NETGEN_INLINE void DeleteElement (IndexType i)
     {
       NETGEN_CHECK_RANGE(i,BASE,BASE+size);
       data[i-BASE] = std::move(data[size-1]);
@@ -956,10 +979,10 @@ namespace ngcore
 
 
     /// Delete element i. Move all remaining elements forward
-    NETGEN_INLINE void RemoveElement (size_t i)
+    NETGEN_INLINE void RemoveElement (IndexType i)
     {
       NETGEN_CHECK_RANGE(i, BASE, BASE+size);
-      for(size_t j = i; j < this->size-1; j++)
+      for(size_t j = i-BASE; j+1 < this->size; j++)
 	this->data[j] = this->data[j+1];
       this->size--;
     }
@@ -1011,8 +1034,7 @@ namespace ngcore
     /// steal array 
     NETGEN_INLINE Array & operator= (Array && a2)
     {
-      mt.Swap(sizeof(T)*allocsize, a2.mt, sizeof(T)*a2.allocsize);
+      mt = std::move(a2.mt);
-
       ngcore::Swap (size, a2.size);
       ngcore::Swap (data, a2.data);
       ngcore::Swap (allocsize, a2.allocsize);
@@ -1086,8 +1108,7 @@ namespace ngcore
     
     NETGEN_INLINE void Swap (Array & b)
     {
-      mt.Swap(sizeof(T) * allocsize, b.mt, sizeof(T) * b.allocsize);
+      mt = std::move(b.mt);
-
       ngcore::Swap (size, b.size);
       ngcore::Swap (data, b.data);
       ngcore::Swap (allocsize, b.allocsize);
@@ -1096,6 +1117,7 @@ namespace ngcore
 
     NETGEN_INLINE void StartMemoryTracing () const
     {
+      if(mem_to_delete)
         mt.Alloc(sizeof(T) * allocsize);
     }
 
@@ -1105,7 +1127,6 @@ namespace ngcore
 
     /// resize array, at least to size minsize. copy contents
     NETGEN_INLINE void ReSize (size_t minsize);
-    MemoryTracer mt;
   };
 
   
@@ -1158,6 +1179,7 @@ namespace ngcore
     using Array<T>::allocsize;
     using Array<T>::data;
     using Array<T>::mem_to_delete;
+    using Array<T>::mt;
     // using Array<T>::ownmem;
 
   public:
@@ -1171,6 +1193,7 @@ namespace ngcore
           data = new T[asize];
           allocsize = size;
           mem_to_delete = data;
+          mt.Alloc(sizeof(T)*asize);
         }
     }
 
@@ -1191,6 +1214,7 @@ namespace ngcore
     ArrayMem(ArrayMem && a2)
       : Array<T> (a2.Size(), (T*)mem)
     {
+      mt = std::move(a2.mt);
       if (a2.mem_to_delete)
         {
           mem_to_delete = a2.mem_to_delete;
@@ -1233,6 +1257,7 @@ namespace ngcore
 
     ArrayMem & operator= (ArrayMem && a2)
     {
+      mt = std::move(a2.mt);
       ngcore::Swap (mem_to_delete, a2.mem_to_delete);
       ngcore::Swap (allocsize, a2.allocsize);
       ngcore::Swap (size, a2.size);
@@ -1528,6 +1553,8 @@ namespace ngcore
   }
                                
 
+  struct HTAHelp { };
+  
   // head-tail array
   template <size_t S, typename T>
   class HTArray
@@ -1535,10 +1562,22 @@ namespace ngcore
     HTArray<S-1,T> tail;
     T head;
   public:
-    HTArray () = default;
+    constexpr HTArray () = default;
-    HTArray (const HTArray &) = default;
+    constexpr HTArray (const HTArray &) = default;
     template <typename T2>
-    HTArray (const HTArray<S,T2> & a2) : tail(a2.Tail()), head(a2.Head()) { ; }
+    constexpr HTArray (const HTArray<S,T2> & a2) : tail(a2.Tail()), head(a2.Head()) { ; }
+
+    constexpr HTArray (T v) : tail(v), head(v) { } // all the same
+    
+    template <class... T2,
+              std::enable_if_t<S==1+sizeof...(T2),bool> = true>
+    constexpr HTArray (const T &v, T2... rest)
+      : tail{HTAHelp(), v,rest...}, head(std::get<S-2>(std::tuple(rest...))) { }
+
+    template <class... T2>
+    constexpr HTArray (HTAHelp h, const T &v, T2... rest)
+      : tail{h, v,rest...}, head(std::get<S-2>(std::tuple(rest...))) { }
+
     
     HTArray & operator= (const HTArray &) = default;
 
@@ -1559,10 +1598,15 @@ namespace ngcore
   {
     T head;
   public:
-    HTArray () = default;
+    constexpr HTArray () = default;
-    HTArray (const HTArray &) = default;
+    constexpr HTArray (const HTArray &) = default;
     template <typename T2>
-    HTArray (const HTArray<1,T2> & a2) : head(a2.Head()) { ; }
+    constexpr HTArray (const HTArray<1,T2> & a2) : head(a2.Head()) { ; }
+    constexpr HTArray (T v) : head(v) { } // all the same
+    template <class... T2>    
+    constexpr HTArray (HTAHelp h, const T &v, T2... rest)
+      : head(v) { } 
+
     
     HTArray & operator= (const HTArray &) = default;
 
@@ -1590,7 +1634,7 @@ namespace ngcore
     HTArray (const HTArray &) = default;
     template <typename T2>
     HTArray (const HTArray<0,T2> & a2) { ; }
-    
+    constexpr HTArray (T v) { } // all the same    
     HTArray & operator= (const HTArray &) = default;
 
     /*

libsrc/core/bitarray.cpp

@@ -40,12 +40,13 @@ namespace ngcore
     if (owns_data)
       {
         delete [] data;
-        mt.Free(Addr(size)+1);
+        mt.Free(GetMemoryUsage());
       }
 
     size = asize;
     data = new unsigned char [Addr (size)+1];
-    mt.Alloc(Addr(size)+1);
+    owns_data = true;
+    mt.Alloc(GetMemoryUsage());
   }
 
   BitArray & BitArray :: Set () throw()

libsrc/core/bitarray.hpp

@@ -49,6 +49,7 @@ public:
   {
     ba2.owns_data = false;
     ba2.data = nullptr;
+    mt = std::move(ba2.mt);
   }
 
   template <typename T>
@@ -59,13 +60,17 @@ public:
     int cnt = 0;
     for (auto i = list.begin(); i < list.end(); i++, cnt++)
       if (*i) SetBit(cnt);
+    StartMemoryTracing();
   }
 
   /// delete data
   ~BitArray ()
   {
     if (owns_data)
+    {
       delete [] data;
+      mt.Free(GetMemoryUsage());
+    }
   }
 
   /// Set size, loose values
@@ -150,11 +155,11 @@ public:
   
   NGCORE_API auto * Data() const { return data; }
 
+  const size_t GetMemoryUsage() const { return owns_data ? (size+CHAR_BIT-1)/CHAR_BIT : 0; }
   const MemoryTracer& GetMemoryTracer() const { return mt; }
   void StartMemoryTracing() const
   {
-    if(owns_data)
+    mt.Alloc(GetMemoryUsage());
-      mt.Alloc(Addr(size)+1);
   }
 
 private:
@@ -205,6 +210,31 @@ private:
 
   NGCORE_API std::ostream & operator<<(std::ostream & s, const BitArray & ba);
 
+
+
+  template <typename IndexType>
+  class TBitArray : public BitArray
+  {
+  public:
+    using BitArray::BitArray;
+
+    void SetBit (IndexType i) { BitArray::SetBit(i-IndexBASE<IndexType>()); }
+    void Clear () { BitArray::Clear(); }
+    void Clear (IndexType i) { BitArray::Clear(i-IndexBASE<IndexType>()); }
+    void SetBitAtomic (IndexType i) { BitArray::SetBitAtomic(i-IndexBASE<IndexType>()); }
+    bool Test (IndexType i) const { return BitArray::Test(i-IndexBASE<IndexType>()); }
+    
+    bool operator[] (IndexType i) const { return Test(i); } 
+    T_Range<IndexType> Range() const { return { IndexBASE<IndexType>(), IndexBASE<IndexType>()+Size() }; }
+    NGCORE_API TBitArray & Or (const TBitArray & ba2)
+    {
+      BitArray::Or(ba2);
+      return *this;
+    }
+
+  };
+
 } // namespace ngcore
 
+  
 #endif // NETGEN_CORE_BITARRAY

libsrc/core/exception.cpp

@@ -23,6 +23,47 @@ namespace ngcore
 }
 
 
+  Exception :: Exception(std::string_view s1, std::string_view s2)
+    : Exception(std::string(s1)+std::string(s2))
+  { }
+
+  Exception :: Exception(std::string_view s1, std::string_view s2, std::string_view s3)
+    : Exception(std::string(s1)+std::string(s2)+std::string(s3))
+  { }
+
+
+  void Exception :: Throw (std::string_view s1)
+  {
+    throw Exception(std::string(s1));
+  }
+
+  void Exception :: Throw (std::string_view s1, std::string_view s2)
+  {
+    throw Exception(std::string(s1)+std::string(s2));
+  }
+    
+  void Exception :: Throw (std::string_view s1, std::string_view s2, std::string_view s3)
+  {
+    throw Exception(std::string(s1)+std::string(s2)+std::string(s3));
+  }
+  
+  
+  RangeException :: RangeException (// const std::string & where,
+                                    const char * where,
+                                    ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax) : Exception("")
+  {
+    std::stringstream str;
+    str << where << ": index " << ind << " out of range [" << imin << "," << imax << ")\n";
+    Append (str.str());
+    Append (GetBackTrace());
+  }
+  
+
+  void ThrowRangeException(const char * s, ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax)
+  {
+    throw RangeException(s, ind, imin, imax);
+  }
+  
   void ThrowException(const std::string & s)
   {
     throw Exception (s);
@@ -32,6 +73,13 @@ namespace ngcore
   {
     throw Exception (s);
   }
+
+
+  void ThrowNotTheSameException(const char * s, ptrdiff_t a, ptrdiff_t b)
+  {
+    throw ngcore::Exception(std::string(s) + ", a="+ToString(a) + ", b="+ToString(b) + GetBackTrace());     
+  }
+  
 } // namespace ngcore
 
 
@@ -80,7 +128,7 @@ namespace ngcore
       // 1   libngcore.dylib                     0x000000010ddb298c _ZL21ngcore_signal_handleri + 316
       constexpr char reset_shell[] = "\033[0m";
       constexpr char green[] = "\033[32m";
-      constexpr char yellow[] = "\033[33m";
+      [[maybe_unused]] constexpr char yellow[] = "\033[33m";
 
       std::istringstream in(s);
 

libsrc/core/exception.hpp

@@ -1,16 +1,17 @@
 #ifndef NETGEN_CORE_EXCEPTION_HPP
 #define NETGEN_CORE_EXCEPTION_HPP
 
+#include <cstddef>
 #include <sstream>         // for stringstream
 #include <stdexcept>       // for exception
 #include <string>          // for string
 
 #include "ngcore_api.hpp"  // for NGCORE_API
+#include "utils.hpp"       // for ToString
 
 
 namespace ngcore
 {
-
   NGCORE_API std::string GetBackTrace();
 
   // Exception for code that shouldn't be executed
@@ -33,8 +34,14 @@ namespace ngcore
     Exception(Exception&&) = default;
     Exception(const std::string& s); //  : m_what(s) {}
     Exception(const char* s); //  : m_what(s) {}
+    Exception(std::string_view s1, std::string_view s2);
+    Exception(std::string_view s1, std::string_view s2, std::string_view s3);
     ~Exception() override = default;
 
+    [[noreturn]] static void Throw (std::string_view s1);
+    [[noreturn]] static void Throw (std::string_view s1, std::string_view s2);
+    [[noreturn]] static void Throw (std::string_view s1, std::string_view s2, std::string_view s3);
+    
     Exception& operator =(const Exception&) = default;
     Exception& operator =(Exception&&) noexcept = default;
 
@@ -50,23 +57,26 @@ namespace ngcore
     const char* what() const noexcept override { return m_what.c_str(); }
   };
   
-  NGCORE_API void ThrowException(const std::string & s);
+  [[noreturn]] NGCORE_API void ThrowException(const std::string & s);
-  NGCORE_API void ThrowException(const char * s);
+  [[noreturn]] NGCORE_API void ThrowException(const char * s);
   
   // Out of Range exception
   class NGCORE_API RangeException : public Exception
   {
   public:
     /// where it occurs, index, minimal and maximal indices
-    RangeException (const std::string & where,
+    RangeException (// const std::string & where,
-                    int ind, int imin, int imax) : Exception("")
+                    const char * where,
+                    ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax);
+    /*
+    : Exception("")
       {
         std::stringstream str;
         str << where << ": index " << ind << " out of range [" << imin << "," << imax << ")\n";
         Append (str.str());
         Append (GetBackTrace());
       }
-
+    */
     template<typename T>
     RangeException(const std::string& where, const T& value)
     {
@@ -76,9 +86,40 @@ namespace ngcore
     }
   };
 
+  [[noreturn]] NGCORE_API void ThrowRangeException(const char * s, ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax);
+  [[noreturn]] NGCORE_API void ThrowNotTheSameException(const char * s, ptrdiff_t a, ptrdiff_t b);
+  
+  
   // Exception used if no simd implementation is available to fall back to standard evaluation
   class NGCORE_API ExceptionNOSIMD : public Exception
   { public: using Exception::Exception; };
+
+  template <typename T>
+  struct IsSafe {
+    constexpr operator bool() const { return false; } };
+
+  namespace detail {
+    template <typename T, typename Tmin, typename Tmax>
+    inline static constexpr void CheckRange(const char * s, const T& n, Tmin first, Tmax next)
+    {
+      if constexpr (!IsSafe<decltype(n)>())
+        if (n<first || n>=next)
+          ThrowRangeException(s, ptrdiff_t(n), ptrdiff_t(first), ptrdiff_t(next));
+    }
+
+    template <typename Ta, typename Tb>
+    inline static constexpr void CheckSame(const char * s, const Ta& a, const Tb& b)
+    {
+     if constexpr (!IsSafe<decltype(a)>() || !IsSafe<decltype(b)>())
+      if(a != b)
+      {
+        if constexpr(std::is_integral_v<decltype(a)> && std::is_same_v<decltype(a),decltype(b)>)
+          ThrowNotTheSameException(s, long(a), long(b)); \
+        else
+          throw Exception(std::string(s) + "\t: not the same, a="+ToString(a) + ", b="+ngcore::ToString(b) + GetBackTrace());
+      }
+    }
+  } // namespace detail
 } // namespace ngcore
 
 #define NETGEN_CORE_NGEXEPTION_STR_HELPER(x) #x
@@ -88,20 +129,14 @@ namespace ngcore
 #define NG_EXCEPTION(s) ngcore::Exception(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t"+std::string(s))
 
 #if defined(NETGEN_ENABLE_CHECK_RANGE) && !defined(__CUDA_ARCH__)
-#define NETGEN_CHECK_RANGE(value, min, max_plus_one) \
+#define NETGEN_CHECK_RANGE(value, min, max_plus_one) ngcore::detail::CheckRange(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t", value, min, max_plus_one);
-  { if ((value)<(min) ||  (value)>=(max_plus_one)) \
+#define NETGEN_CHECK_SAME(a,b) ngcore::detail::CheckSame(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t", a, b);
-      throw ngcore::RangeException(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t", int(value), int(min), int(max_plus_one)); }
+
-#define NETGEN_CHECK_SHAPE(a,b) \
-  { if(a.Shape() != b.Shape()) \
-      throw ngcore::Exception(__FILE__": shape don't match"); }
-#define NETGEN_CHECK_SAME(a,b) \
-  { if(a != b) \
-      throw ngcore::Exception(__FILE__": not the same, a="+ToString(a) + ", b="+ToString(b)); }
 #define NETGEN_NOEXCEPT 
 #else // defined(NETGEN_ENABLE_CHECK_RANGE) && !defined(__CUDA_ARCH__)
 #define NETGEN_CHECK_RANGE(value, min, max)
 #define NETGEN_CHECK_SAME(a,b)
-#define NETGEN_CHECK_SHAPE(a,b)
+// #define NETGEN_CHECK_SHAPE(a,b)
 #define NETGEN_NOEXCEPT noexcept
 #endif // defined(NETGEN_ENABLE_CHECK_RANGE) && !defined(__CUDA_ARCH__)
 

libsrc/core/flags.cpp

@@ -16,6 +16,7 @@
 namespace ngcore
 {
   using std::string;
+  using std::string_view;
   using std::endl;
   Flags :: Flags () { ; }
 
@@ -209,18 +210,18 @@ namespace ngcore
   }
 
 
-  double Flags :: GetNumFlag (const string & name, double def) const
+  double Flags :: GetNumFlag (string_view name, double def) const
   {
     if (numflags.Used (name))
-      return numflags[name];
+      return numflags[string(name)];
     else
       return def;
   }
   
-  const double * Flags :: GetNumFlagPtr (const string & name) const
+  const double * Flags :: GetNumFlagPtr (string_view name) const
   {
     if (numflags.Used (name))
-      return & ((SymbolTable<double>&)numflags)[name];
+      return & ((SymbolTable<double>&)numflags)[string(name)];
     else
       return NULL;
   }
@@ -239,16 +240,16 @@ namespace ngcore
     return defflags.Used (name);
   }
   */
-  bool Flags :: GetDefineFlag (const string & name) const throw()
+  bool Flags :: GetDefineFlag (string_view name) const throw()
   {
-    if (!defflags.Used (name)) return false;
+    if (!defflags.Used (string(name))) return false;
-    return defflags[name];
+    return defflags[string(name)];
   }
 
-  xbool Flags :: GetDefineFlagX (const string & name) const throw()
+  xbool Flags :: GetDefineFlagX (string_view name) const throw()
   {
-    if (!defflags.Used (name)) return maybe;
+    if (!defflags.Used (string(name))) return maybe;
-    return bool(defflags[name]);
+    return bool(defflags[string(name)]);
   }
 
 
@@ -296,32 +297,32 @@ namespace ngcore
     return empty;
   }
 
-  bool Flags :: StringFlagDefined (const string & name) const
+  bool Flags :: StringFlagDefined (string_view name) const noexcept
   {
     return strflags.Used (name);
   }
 
-  bool Flags :: NumFlagDefined (const string &name) const
+  bool Flags :: NumFlagDefined (string_view name) const noexcept
   {
     return numflags.Used (name);
   }
 
-  bool Flags :: FlagsFlagDefined (const string &name) const
+  bool Flags :: FlagsFlagDefined (string_view name) const noexcept
   {
     return flaglistflags.Used (name);
   }
   
-  bool Flags :: StringListFlagDefined (const string & name) const
+  bool Flags :: StringListFlagDefined (string_view name) const noexcept
   {
     return strlistflags.Used (name);
   }
 
-  bool Flags :: NumListFlagDefined (const string & name) const
+  bool Flags :: NumListFlagDefined (string_view name) const noexcept
   {
     return numlistflags.Used (name);
   }
 
-  bool Flags :: AnyFlagDefined (const string& name) const
+  bool Flags :: AnyFlagDefined (string_view name) const noexcept
   {
     return anyflags.Used(name);
   }

libsrc/core/flags.hpp

@@ -125,15 +125,15 @@ namespace ngcore
     /// Returns std::string flag, default value if not exists
     std::string GetStringFlag (const std::string & name, std::string def = "") const;
     /// Returns numerical flag, default value if not exists
-    double GetNumFlag (const std::string & name, double def) const;
+    double GetNumFlag (std::string_view name, double def) const;
     /// Returns address of numerical flag, null if not exists
-    const double * GetNumFlagPtr (const std::string & name) const;
+    const double * GetNumFlagPtr (std::string_view name) const;
     /// Returns address of numerical flag, null if not exists
     double * GetNumFlagPtr (const std::string & name);
     /// Returns boolean flag
     // int GetDefineFlag (const char * name) const;
-    bool GetDefineFlag (const std::string & name) const throw();
+    bool GetDefineFlag (std::string_view name) const  noexcept;
-    xbool GetDefineFlagX (const std::string & name) const throw();
+    xbool GetDefineFlagX (std::string_view name) const  noexcept;
     /// Returns string list flag, empty array if not exist
     const Array<std::string> & GetStringListFlag (const std::string & name) const;
     /// Returns num list flag, empty array if not exist
@@ -144,16 +144,16 @@ namespace ngcore
 
 
     /// Test, if string flag is defined
-    bool StringFlagDefined (const std::string & name) const;
+    bool StringFlagDefined (std::string_view name) const noexcept;
     /// Test, if num flag is defined
-    bool NumFlagDefined (const std::string & name) const;
+    bool NumFlagDefined (std::string_view name) const noexcept;
     /// Test, if num flag is defined
-    bool FlagsFlagDefined (const std::string & name) const;
+    bool FlagsFlagDefined (std::string_view name) const noexcept;
     /// Test, if string list flag is defined
-    bool StringListFlagDefined (const std::string & name) const;
+    bool StringListFlagDefined (std::string_view name) const noexcept;
     /// Test, if num list flag is defined
-    bool NumListFlagDefined (const std::string & name) const;
+    bool NumListFlagDefined (std::string_view name) const noexcept;
-    bool AnyFlagDefined (const std::string& name) const;
+    bool AnyFlagDefined (std::string_view name) const noexcept;
 
     /// number of string flags
     int GetNStringFlags () const { return strflags.Size(); }

libsrc/core/generate_mpi_sources.py

@@ -0,0 +1,174 @@
+functions = [
+        ("double", "MPI_Wtime"),
+        ("int", "MPI_Allgather", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "MPI_Comm"),
+        ("int", "MPI_Allreduce", "void*", "void*", "int", "MPI_Datatype", "MPI_Op", "MPI_Comm"),
+        ("int", "MPI_Alltoall", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "MPI_Comm"),
+        ("int", "MPI_Barrier", "MPI_Comm"),
+        ("int", "MPI_Bcast", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Ibcast", "void*", "int", "MPI_Datatype", "int", "MPI_Comm", "MPI_Request*"),    
+        ("int", "MPI_Comm_c2f", "MPI_Comm"),
+        ("int", "MPI_Comm_create", "MPI_Comm", "MPI_Group", "MPI_Comm*"),
+        ("int", "MPI_Comm_create_group", "MPI_Comm", "MPI_Group", "int", "MPI_Comm*"),
+        ("int", "MPI_Comm_free", "MPI_Comm*"),
+        ("int", "MPI_Comm_group", "MPI_Comm", "MPI_Group*"),
+        ("int", "MPI_Comm_rank", "MPI_Comm", "int*"),
+        ("int", "MPI_Comm_size", "MPI_Comm", "int*"),
+        ("int", "MPI_Finalize"),
+        ("int", "MPI_Gather", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Gatherv", "void*", "int", "MPI_Datatype", "void*", "int*", "int*", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Get_count", "MPI_Status*", "MPI_Datatype", "int*"),
+        ("int", "MPI_Get_processor_name", "char*", "int*"),
+        ("int", "MPI_Group_incl", "MPI_Group", "int", "int*", "MPI_Group*"),
+        ("int", "MPI_Init", "int*", "char***"),
+        ("int", "MPI_Init_thread", "int*", "char***", "int", "int*"),
+        ("int", "MPI_Initialized", "int*"),
+        ("int", "MPI_Iprobe", "int", "int", "MPI_Comm", "int*", "MPI_Status*"),
+        ("int", "MPI_Irecv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Isend", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Probe", "int", "int", "MPI_Comm", "MPI_Status*"),
+        ("int", "MPI_Query_thread", "int*"),
+        ("int", "MPI_Recv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Status*"),
+        ("int", "MPI_Recv_init", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Reduce", "void*", "void*", "int", "MPI_Datatype", "MPI_Op", "int", "MPI_Comm"),
+        ("int", "MPI_Reduce_local", "void*", "void*", "int", "MPI_Datatype", "MPI_Op"),
+        ("int", "MPI_Request_free", "MPI_Request*"),
+        ("int", "MPI_Scatter", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Send", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm"),
+        ("int", "MPI_Send_init", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Startall", "int", "MPI_Request*:0"),
+        ("int", "MPI_Type_commit", "MPI_Datatype*"),
+        ("int", "MPI_Type_contiguous", "int", "MPI_Datatype", "MPI_Datatype*"),
+        ("int", "MPI_Type_create_resized", "MPI_Datatype", "MPI_Aint", "MPI_Aint", "MPI_Datatype*"),
+        ("int", "MPI_Type_create_struct", "int", "int*:0", "MPI_Aint*:0", "MPI_Datatype*:0", "MPI_Datatype*"),
+        ("int", "MPI_Type_free", "MPI_Datatype*"),
+        ("int", "MPI_Type_get_extent", "MPI_Datatype", "MPI_Aint*", "MPI_Aint*"),
+        ("int", "MPI_Type_indexed", "int", "int*:0", "int*:0", "MPI_Datatype", "MPI_Datatype*"),
+        ("int", "MPI_Type_size", "MPI_Datatype", "int*"),
+        ("int", "MPI_Wait", "MPI_Request*", "MPI_Status*"),
+        ("int", "MPI_Waitall", "int", "MPI_Request*:0", "MPI_Status*"),
+        ("int", "MPI_Waitany", "int", "MPI_Request*:0", "int*", "MPI_Status*"),
+        ]
+
+constants = [
+        ("MPI_Comm", "MPI_COMM_NULL"),
+        ("MPI_Comm", "MPI_COMM_WORLD"),
+        ("MPI_Datatype", "MPI_CHAR"),
+        ("MPI_Datatype", "MPI_CXX_DOUBLE_COMPLEX"),
+        ("MPI_Datatype", "MPI_C_BOOL"),
+        ("MPI_Datatype", "MPI_DATATYPE_NULL"),
+        ("MPI_Datatype", "MPI_DOUBLE"),
+        ("MPI_Datatype", "MPI_FLOAT"),
+        ("MPI_Datatype", "MPI_INT"),
+        ("MPI_Datatype", "MPI_SHORT"),
+        ("MPI_Datatype", "MPI_UINT64_T"),
+        ("MPI_Op", "MPI_LOR"),
+        ("MPI_Op", "MPI_MAX"),
+        ("MPI_Op", "MPI_MIN"),
+        ("MPI_Op", "MPI_SUM"),
+        ("MPI_Request", "MPI_REQUEST_NULL"),
+        ("MPI_Status*", "MPI_STATUSES_IGNORE"),
+        ("MPI_Status*", "MPI_STATUS_IGNORE"),
+        ("int", "MPI_ANY_SOURCE"),
+        ("int", "MPI_ANY_TAG"),
+        ("int", "MPI_MAX_PROCESSOR_NAME"),
+        ("int", "MPI_PROC_NULL"),
+        ("int", "MPI_ROOT"),
+        ("int", "MPI_SUBVERSION"),
+        ("int", "MPI_THREAD_MULTIPLE"),
+        ("int", "MPI_THREAD_SINGLE"),
+        ("int", "MPI_VERSION"),
+        ("void*", "MPI_IN_PLACE"),
+]
+
+def get_args(f, counts=False):
+    args = []
+    for arg in f[2:]:
+        has_count = ':' in arg
+        if has_count:
+            s, count = arg.split(':')
+            count = int(count)
+        else:
+            s = arg
+            count = None
+        if s.startswith("MPI_"):
+            s = "NG_" + s
+        if counts:
+            args.append((s, count))
+        else:
+            args.append(s)
+    return args
+
+def generate_declarations():
+    code = ""
+    nowrapper_code = ""
+    for f in functions:
+        ret = f[0]
+        name = f[1]
+        args = ", ".join(get_args(f))
+        code += f"NGCORE_API extern {ret} (*NG_{name})({args});\n"
+        nowrapper_code += f"#define NG_{name} {name}\n"
+
+    for typ, name in constants:
+        if typ.startswith("MPI_"):
+            typ = "NG_" + typ
+        code += f"NGCORE_API extern {typ} NG_{name};\n"
+        nowrapper_code += f"#define NG_{name} {name}\n"
+
+    with open("ng_mpi_generated_declarations.hpp", "w") as f:
+        f.write("#ifdef NG_MPI_WRAPPER\n")
+        f.write(code)
+        f.write("#else  // NG_MPI_WRAPPER\n")
+        f.write(nowrapper_code)
+        f.write("#endif // NG_MPI_WRAPPER\n")
+
+def generate_dummy_init():
+    code = ""
+    for f in functions:
+        ret = f[0]
+        name = f[1]
+        args = ", ".join(get_args(f))
+        code += f"decltype(NG_{name}) NG_{name} = []({args})->{ret} {{ throw no_mpi(); }};\n"
+
+    for typ, name in constants:
+        if typ.startswith("MPI_"):
+            typ = "NG_" + typ
+        code += f"{typ} NG_{name} = 0;\n"
+
+    with open("ng_mpi_generated_dummy_init.hpp", "w") as f:
+        f.write(code)
+
+def generate_init():
+    code = ""
+    for f in functions:
+        ret = f[0]
+        name = f[1]
+        args = get_args(f, counts=True)
+        in_args  =''
+        call_args = ''
+        for i in range(len(args)):
+            arg, count = args[i]
+            if i > 0:
+                in_args += ', '
+                call_args += ', '
+            in_args += arg + f" arg{i}"
+            if not arg.startswith("NG_"):
+                # plain type (like int, int *, etc.), just pass the argument along
+                call_args += f" arg{i}"
+            elif count is None:
+                # MPI type (by value or pointer), but just one object, no arrays
+                call_args += f" ng2mpi(arg{i})"
+            else:
+                # arrays of MPI types, we need to copy them due to incompatible size
+                call_args += f" ng2mpi(arg{i}, arg{count})"
+        code += f"NG_{name} = []({in_args})->{ret} {{ return {name}({call_args}); }};\n"
+
+    for _, name in constants:
+        code += f"NG_{name} = mpi2ng({name});\n"
+
+    with open("ng_mpi_generated_init.hpp", "w") as f:
+        f.write(code)
+
+if __name__ == "__main__":
+    generate_declarations()
+    generate_dummy_init()
+    generate_init()

libsrc/core/hashtable.hpp

@@ -9,6 +9,7 @@
 
 #include <string>
 #include <tuple>
+#include <optional>
 
 // #include "mpi_wrapper.hpp"
 #include "ngcore_api.hpp"
@@ -38,52 +39,67 @@ namespace ngcore
   
   
 
+  // feature check macro for transition from INT to IVec
+#define NGCORE_HAS_IVEC
   
   /// N integers
   template <int N, typename T = int>
-  class INT
+  class IVec
   {
     /// data
-    T i[(N>0)?N:1];
+    // T i[(N>0)?N:1];
+
+    HTArray<N,T> i;
     
   public:
     ///
-    NETGEN_INLINE INT () { }
+    constexpr NETGEN_INLINE IVec () = default;
+    constexpr NETGEN_INLINE IVec (const IVec & i1) : i(i1.i) { }
 
+    constexpr NETGEN_INLINE IVec (T ai1) : i(ai1) { }
+    
+    template <class... T2,
+              std::enable_if_t<N==1+sizeof...(T2),bool> = true>
+    constexpr IVec (const T &v, T2... rest)
+      : i{v,rest...} { } 
+
+    /*
     /// init all
-    NETGEN_INLINE INT (T ai1)
+    NETGEN_INLINE IVec (T ai1)
     { 
      for (int j = 0; j < N; j++) { i[j] = ai1; }
     }
 
     /// init i[0], i[1]
-    constexpr NETGEN_INLINE INT (T ai1, T ai2)
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2)
       : i{ai1, ai2} { ; } 
 
     /// init i[0], i[1], i[2]
-    constexpr NETGEN_INLINE INT (T ai1, T ai2, T ai3)
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2, T ai3)
       : i{ai1, ai2, ai3} { ; } 
 
     /// init i[0], i[1], i[2]
-    constexpr NETGEN_INLINE INT (T ai1, T ai2, T ai3, T ai4)
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2, T ai3, T ai4)
       : i{ai1, ai2, ai3, ai4} { ; }
     
     /// init i[0], i[1], i[2]
-    constexpr NETGEN_INLINE INT (T ai1, T ai2, T ai3, T ai4, T ai5)
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2, T ai3, T ai4, T ai5)
       : i{ai1, ai2, ai3, ai4, ai5} { ; }      
       
     /// init i[0], i[1], i[2]
-    NETGEN_INLINE INT (T ai1, T ai2, T ai3, T ai4, T ai5, T ai6, T ai7, T ai8, T ai9)
+    NETGEN_INLINE IVec (T ai1, T ai2, T ai3, T ai4, T ai5, T ai6, T ai7, T ai8, T ai9)
       : i{ai1, ai2, ai3, ai4, ai5, ai6, ai7, ai8, ai9 } { ; }            
+    */
     
     template <typename ARCHIVE>
     void DoArchive(ARCHIVE& ar)
     {
-      ar.Do(i, N);
+      // ar.Do(i.begin(), N);
+      ar.Do(i.Ptr(), N);
     }
 
     template <int N2, typename T2>
-    NETGEN_INLINE INT (const INT<N2,T2> & in2)
+    NETGEN_INLINE IVec (const IVec<N2,T2> & in2)
     {
       if (N2 <= N)
         {
@@ -100,7 +116,7 @@ namespace ngcore
     }
 
     template <typename T2>
-    NETGEN_INLINE INT (const BaseArrayObject<T2> & ao)
+    NETGEN_INLINE IVec (const BaseArrayObject<T2> & ao)
     {
       for (int j = 0; j < N; j++)
         i[j] = ao.Spec()[j];
@@ -108,7 +124,7 @@ namespace ngcore
     
     NETGEN_INLINE size_t Size() const { return N; }
     /// all ints equal ?
-    NETGEN_INLINE bool operator== (const INT & in2) const
+    NETGEN_INLINE bool operator== (const IVec & in2) const
     { 
       for (int j = 0; j < N; j++) 
 	if (i[j] != in2.i[j]) return 0;
@@ -116,7 +132,7 @@ namespace ngcore
     }
 
     /// any ints unequal ?
-    NETGEN_INLINE bool operator!= (const INT & in2) const
+    NETGEN_INLINE bool operator!= (const IVec & in2) const
     {
       for (int j = 0; j < N; j++)
         if (i[j] != in2.i[j]) return 1;
@@ -124,7 +140,7 @@ namespace ngcore
     }
 
     /// sort integers
-    NETGEN_INLINE INT & Sort () & 
+    NETGEN_INLINE IVec & Sort () & 
     {
       for (int k = 0; k < N; k++)
 	for (int l = k+1; l < N; l++)
@@ -133,7 +149,7 @@ namespace ngcore
       return *this;
     }
 
-    NETGEN_INLINE INT Sort () &&
+    NETGEN_INLINE IVec Sort () &&
     {
       for (int k = 0; k < N; k++)
 	for (int l = k+1; l < N; l++)
@@ -155,7 +171,7 @@ namespace ngcore
     
     operator FlatArray<T> () { return FlatArray<T> (N, &i[0]); } 
 
-    NETGEN_INLINE INT<N,T> & operator= (T value)
+    NETGEN_INLINE IVec<N,T> & operator= (T value)
     {
       for (int j = 0; j < N; j++)
 	i[j] = value;
@@ -163,7 +179,7 @@ namespace ngcore
     }
 
     template <typename T2>
-    NETGEN_INLINE INT<N,T> & operator= (INT<N,T2> v2)
+    NETGEN_INLINE IVec<N,T> & operator= (IVec<N,T2> v2)
     {
       for (int j = 0; j < N; j++)
 	i[j] = v2[j];
@@ -186,14 +202,14 @@ namespace ngcore
 
   /// sort 2 integers
   template <>
-  NETGEN_INLINE INT<2> & INT<2>::Sort () & 
+  NETGEN_INLINE IVec<2> & IVec<2>::Sort () & 
   {
     if (i[0] > i[1]) Swap (i[0], i[1]);
     return *this;
   }
 
   template <>
-  NETGEN_INLINE INT<2> INT<2>::Sort () &&
+  NETGEN_INLINE IVec<2> IVec<2>::Sort () &&
   {
     if (i[0] > i[1]) Swap (i[0], i[1]);
     return *this;
@@ -201,7 +217,7 @@ namespace ngcore
 
   /// sort 3 integers
   template <>
-  NETGEN_INLINE INT<3> INT<3>::Sort () &&
+  NETGEN_INLINE IVec<3> IVec<3>::Sort () &&
   {
     if (i[0] > i[1]) Swap (i[0], i[1]);
     if (i[1] > i[2]) Swap (i[1], i[2]);
@@ -211,7 +227,7 @@ namespace ngcore
 
   /// Print integers
   template <int N, typename T>
-  inline ostream & operator<<(ostream  & s, const INT<N,T> & i2)
+  inline ostream & operator<<(ostream  & s, const IVec<N,T> & i2)
   {
     for (int j = 0; j < N; j++)
       s << (int) i2[j] << " ";
@@ -219,15 +235,15 @@ namespace ngcore
   }
   
   template <int N, typename T>
-  auto begin(const INT<N,T> & ind)
+  auto begin(const IVec<N,T> & ind)
   {
-    return AOWrapperIterator<INT<N,T>> (ind, 0);
+    return AOWrapperIterator<IVec<N,T>> (ind, 0);
   }
 
   template <int N, typename T>
-  auto end(const INT<N,T> & ind)
+  auto end(const IVec<N,T> & ind)
   {
-    return AOWrapperIterator<INT<N,T>> (ind, N);    
+    return AOWrapperIterator<IVec<N,T>> (ind, N);    
   }
 
 
@@ -236,9 +252,9 @@ namespace ngcore
 
   
   template <int N, typename TI>
-  NETGEN_INLINE size_t HashValue (const INT<N,TI> & ind, size_t size)
+  NETGEN_INLINE size_t HashValue (const IVec<N,TI> & ind, size_t size)
   {
-    INT<N,size_t> lind = ind;    
+    IVec<N,size_t> lind = ind;    
     size_t sum = 0;
     for (int i = 0; i < N; i++)
       sum += lind[i];
@@ -247,24 +263,24 @@ namespace ngcore
 
   /// hash value of 1 int
   template <typename TI>
-  NETGEN_INLINE size_t HashValue (const INT<1,TI> & ind, size_t size) 
+  NETGEN_INLINE size_t HashValue (const IVec<1,TI> & ind, size_t size) 
   {
     return ind[0] % size;
   }
 
   /// hash value of 2 int
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue (const INT<2,TI> & ind, size_t size) 
+  NETGEN_INLINE size_t HashValue (const IVec<2,TI> & ind, size_t size) 
   {
-    INT<2,size_t> lind = ind;
+    IVec<2,size_t> lind = ind;
     return (113*lind[0]+lind[1]) % size;
   }
 
   /// hash value of 3 int
   template <typename TI>    
-  NETGEN_INLINE size_t HashValue (const INT<3,TI> & ind, size_t size) 
+  NETGEN_INLINE size_t HashValue (const IVec<3,TI> & ind, size_t size) 
   {
-    INT<3,size_t> lind = ind;
+    IVec<3,size_t> lind = ind;
     return (113*lind[0]+59*lind[1]+lind[2]) % size;
   }
 
@@ -284,9 +300,9 @@ namespace ngcore
 
   
   template <int N, typename TI>
-  NETGEN_INLINE size_t HashValue2 (const INT<N,TI> & ind, size_t mask)
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<N,TI> & ind, size_t mask)
   {
-    INT<N,size_t> lind = ind;    
+    IVec<N,size_t> lind = ind;    
     size_t sum = 0;
     for (int i = 0; i < N; i++)
       sum += lind[i];
@@ -295,32 +311,32 @@ namespace ngcore
 
   /// hash value of 1 int
   template <typename TI>
-  NETGEN_INLINE size_t HashValue2 (const INT<1,TI> & ind, size_t mask) 
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<1,TI> & ind, size_t mask) 
   {
     return ind[0] & mask;
   }
 
   /// hash value of 2 int
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue2 (const INT<2,TI> & ind, size_t mask) 
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<2,TI> & ind, size_t mask) 
   {
-    INT<2,size_t> lind = ind;
+    IVec<2,size_t> lind = ind;
     return (113*lind[0]+lind[1]) & mask;
   }
 
   /// hash value of 3 int
   template <typename TI>    
-  NETGEN_INLINE size_t HashValue2 (const INT<3,TI> & ind, size_t mask) 
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<3,TI> & ind, size_t mask) 
   {
-    INT<3,size_t> lind = ind;
+    IVec<3,size_t> lind = ind;
     return (113*lind[0]+59*lind[1]+lind[2]) & mask;
   }
 
-  NETGEN_INLINE size_t HashValue2 (size_t ind, size_t mask)
+  NETGEN_INLINE constexpr size_t HashValue2 (size_t ind, size_t mask)
   {
     return ind & mask;
   }
-  NETGEN_INLINE size_t HashValue2 (int ind, size_t mask)
+  NETGEN_INLINE constexpr size_t HashValue2 (int ind, size_t mask)
   {
     return size_t(ind) & mask;
   }
@@ -332,7 +348,7 @@ namespace ngcore
   // using ngstd::max;
 
   template <int D, typename T>
-  NETGEN_INLINE T Max (const INT<D,T> & i)
+  NETGEN_INLINE T Max (const IVec<D,T> & i)
   {
     if (D == 0) return 0;
     T m = i[0];
@@ -342,7 +358,7 @@ namespace ngcore
   }
 
   template <int D, typename T>
-  NETGEN_INLINE T Min (const INT<D,T> & i)
+  NETGEN_INLINE T Min (const IVec<D,T> & i)
   {
     if (D == 0) return 0;
     T m = i[0];
@@ -352,18 +368,18 @@ namespace ngcore
   }
 
   template <int D, typename T>
-  NETGEN_INLINE INT<D,T> Max (INT<D,T> i1, INT<D,T> i2)
+  NETGEN_INLINE IVec<D,T> Max (IVec<D,T> i1, IVec<D,T> i2)
   {
-    INT<D,T> tmp;
+    IVec<D,T> tmp;
     for (int i = 0; i < D; i++)
       tmp[i] = std::max(i1[i], i2[i]);
     return tmp;
   }
 
   template <int D, typename T>
-  NETGEN_INLINE INT<D,T> operator+ (INT<D,T> i1, INT<D,T> i2)
+  NETGEN_INLINE IVec<D,T> operator+ (IVec<D,T> i1, IVec<D,T> i2)
   {
-    INT<D,T> tmp;
+    IVec<D,T> tmp;
     for (int i = 0; i < D; i++)
       tmp[i] = i1[i]+i2[i];
     return tmp;
@@ -575,6 +591,26 @@ namespace ngcore
     return res; 
   }
 
+  template <typename T>
+  constexpr inline T InvalidHash() { return T(-1); }
+
+  template <typename T_HASH>
+  struct CHT_trait
+  {
+    constexpr static inline T_HASH Invalid() { return InvalidHash<T_HASH>(); }
+    constexpr static inline size_t HashValue (const T_HASH & hash, size_t mask) { return HashValue2(hash, mask); }
+  };
+
+  template <typename T1, typename T2>
+  struct CHT_trait<std::tuple<T1,T2>>
+  {
+    constexpr static inline std::tuple<T1,T2> Invalid() { return { CHT_trait<T1>::Invalid(), CHT_trait<T2>::Invalid() } ; }
+    constexpr static inline size_t HashValue (const std::tuple<T1,T2> & hash, size_t mask)
+    {
+      return (CHT_trait<T1>::HashValue(std::get<0>(hash), mask) + CHT_trait<T2>::HashValue(std::get<1>(hash),mask)) & mask;
+    }
+  };
+
   
 
   /**
@@ -596,14 +632,18 @@ namespace ngcore
     ///
     Array<T> cont;
     ///
-    T_HASH invalid = -1;
+    // T_HASH invalid = -1;
+    // static constexpr T_HASH invalid = InvalidHash<T_HASH>();
+    static constexpr T_HASH invalid = CHT_trait<T_HASH>::Invalid();
   public:
     ///
     ClosedHashTable (size_t asize = 128)
       : size(RoundUp2(asize)), hash(size), cont(size)
     {
       mask = size-1;
-      hash = T_HASH(invalid);
+      // hash = T_HASH(invalid);
+      // hash = InvalidHash<T_HASH>();
+      hash = CHT_trait<T_HASH>::Invalid();
     }
 
     ClosedHashTable (ClosedHashTable && ht2) = default;
@@ -612,7 +652,8 @@ namespace ngcore
     ClosedHashTable (size_t asize, LocalHeap & lh)
       : size(RoundUp2(asize)), mask(size-1), hash(size, lh), cont(size, lh)
     {
-      hash = T_HASH(invalid);
+      // hash = T_HASH(invalid);
+      hash = InvalidHash<T_HASH>();
     }
 
     ClosedHashTable & operator= (ClosedHashTable && ht2) = default;
@@ -637,7 +678,8 @@ namespace ngcore
 
     size_t Position (const T_HASH ind) const
     {
-      size_t i = HashValue2(ind, mask);
+      // size_t i = HashValue2(ind, mask);
+      size_t i = CHT_trait<T_HASH>::HashValue(ind, mask);
       while (true)
 	{
 	  if (hash[i] == ind) return i;
@@ -659,7 +701,8 @@ namespace ngcore
     {
       if (UsedElements()*2 > Size()) DoubleSize();
       
-      size_t i = HashValue2 (ind, mask);
+      // size_t i = HashValue2 (ind, mask);
+      size_t i = CHT_trait<T_HASH>::HashValue (ind, mask);
 
       while (true)
 	{
@@ -704,6 +747,16 @@ namespace ngcore
       return (Position (ahash) != size_t(-1));
     }
 
+    inline std::optional<T> GetIfUsed (const T_HASH & ahash) const
+    {
+      size_t pos = Position (ahash);
+      if (pos != size_t(-1))
+        return cont[pos];
+      else
+        return std::nullopt;
+    }
+    
+
     void SetData (size_t pos, const T_HASH & ahash, const T & acont)
     {
       hash[pos] = ahash;
@@ -782,6 +835,15 @@ namespace ngcore
       used = 0;
     }
 
+    template <typename ARCHIVE>
+    void DoArchive (ARCHIVE& ar)
+    {
+      ar & hash & cont;
+      ar & size & mask & used;
+    }    
+
+    struct EndIterator { };
+    
     class Iterator
     {
       const ClosedHashTable & tab;
@@ -798,24 +860,21 @@ namespace ngcore
         while (nr < tab.Size() && !tab.UsedPos(nr)) nr++;
         return *this;
       }
-      bool operator!= (const Iterator & it2) { return nr != it2.nr; }
+
-      auto operator* () const
+      bool operator!= (EndIterator it2) { return nr != tab.Size(); }
-      {
+      
-        T_HASH hash;
+      auto operator* () const { return tab.GetBoth(nr); }
-        T val;
-        tab.GetData(nr, hash,val);
-        return std::make_pair(hash,val);
-      }
     };
 
     Iterator begin() const { return Iterator(*this, 0); }
-    Iterator end() const { return Iterator(*this, Size()); } 
+    EndIterator end() const { return EndIterator(); }
   };
 
   template <class T_HASH, class T>  
   ostream & operator<< (ostream & ost,
                         const ClosedHashTable<T_HASH,T> & tab)
   {
+    /*
     for (size_t i = 0; i < tab.Size(); i++)
       if (tab.UsedPos(i))
         {
@@ -824,25 +883,28 @@ namespace ngcore
           tab.GetData (i, key, val);
           ost << key << ": " << val << ", ";
         }
+    */
+    for (auto [key,val] : tab)
+      ost << key << ": " << val << ", ";      
     return ost;
   }
 
   template <typename TI>
-  NETGEN_INLINE size_t HashValue (const INT<3,TI> ind)
+  NETGEN_INLINE size_t HashValue (const IVec<3,TI> ind)
   {
-    INT<3,size_t> lind = ind;
+    IVec<3,size_t> lind = ind;
     return 113*lind[0]+59*lind[1]+lind[2];
   }
 
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue (const INT<2,TI> ind)
+  NETGEN_INLINE size_t HashValue (const IVec<2,TI> ind)
   {
-    INT<2,size_t> lind = ind;
+    IVec<2,size_t> lind = ind;
     return 113*lind[0]+lind[1];
   }
 
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue (const INT<1,TI> ind)
+  NETGEN_INLINE size_t HashValue (const IVec<1,TI> ind)
   {
     return ind[0];
   }
@@ -1068,6 +1130,106 @@ namespace ngcore
     return ost;
   }
 
+
+
+
+
+
+
+
+
+  template <class T, class IndexType>
+  class CompressedTable
+  {
+    Table<T, size_t> table;
+    ClosedHashTable<IndexType, size_t> idmap;
+    
+  public:
+    CompressedTable (Table<T, size_t> && atable, ClosedHashTable<IndexType, size_t> && aidmap)
+      : table(std::move(atable)), idmap(std::move(aidmap)) { }
+
+    FlatArray<T> operator[](IndexType id) const
+    {
+      if (auto nr = idmap.GetIfUsed(id))
+        return table[*nr];
+      else
+        return { 0, nullptr };
+    }
+    auto & GetTable() { return table; }
+  };
+
+
+  template <class T, typename IndexType>
+  class CompressedTableCreator
+  {
+  protected:
+    int mode;    // 1 .. cnt, 2 .. cnt entries, 3 .. fill table
+    size_t nd;   // number of entries;
+    ClosedHashTable<IndexType, size_t> idmap;
+    Array<int,size_t> cnt;
+    Table<T,size_t> table;
+  public:
+    CompressedTableCreator()
+    { nd = 0; mode = 1; }
+
+    CompressedTable<T,IndexType> MoveTable()
+    {
+      return { std::move(table), std::move(idmap) };
+    }
+
+    bool Done () { return mode > 3; }
+    void operator++(int) { SetMode (mode+1); }
+
+    int GetMode () const { return mode; }
+    void SetMode (int amode)
+    {
+      mode = amode;
+      if (mode == 2)
+	{
+          cnt.SetSize(nd);  
+          cnt = 0;
+	}
+      if (mode == 3)
+	{
+          table = Table<T,size_t> (cnt);
+          cnt = 0;
+	}
+    }
+
+    void Add (IndexType blocknr, const T & data)
+    {
+      switch (mode)
+	{
+	case 1:
+          {
+            if (!idmap.Used (blocknr))
+              idmap[blocknr] = nd++;
+            break;
+          }
+	case 2:
+	  cnt[idmap.Get(blocknr)]++;
+	  break;
+	case 3:
+          size_t cblock = idmap.Get(blocknr);
+          int ci = cnt[cblock]++;
+          table[cblock][ci] = data;
+	  break;
+	}
+    }
+  };
+
+
+  
+
+
+
+
+
+
+
+
+
+  
 } // namespace ngcore
 
 
@@ -1075,7 +1237,7 @@ namespace ngcore
 #ifdef PARALLEL
 namespace ngcore {
   template<int S, typename T>
-  class MPI_typetrait<ngcore::INT<S, T> >
+  class MPI_typetrait<ngcore::IVec<S, T> >
   {
   public:
     /// gets the MPI datatype
@@ -1099,7 +1261,7 @@ namespace ngcore
   template<typename T> struct MPI_typetrait;
   
   template<int S, typename T>
-  struct MPI_typetrait<INT<S, T> > {
+  struct MPI_typetrait<IVec<S, T> > {
     static auto MPIType () {
       return MPI_typetrait<std::array<T,S>>::MPIType();
     }
@@ -1112,8 +1274,8 @@ namespace std
 {
   // structured binding support
   template <auto N, typename T>
-  struct tuple_size<ngcore::INT<N,T>> : std::integral_constant<std::size_t, N> {};
+  struct tuple_size<ngcore::IVec<N,T>> : std::integral_constant<std::size_t, N> {};
-  template<size_t N, auto M, typename T> struct tuple_element<N,ngcore::INT<M,T>> { using type = T; };
+  template<size_t N, auto M, typename T> struct tuple_element<N,ngcore::IVec<M,T>> { using type = T; };
 }
 
 #endif

libsrc/core/logging.hpp

@@ -42,7 +42,12 @@ namespace ngcore
     static NGCORE_API level::level_enum global_level;
 
   public:
-    static void SetGlobalLoggingLevel( level::level_enum level ) { global_level = level; }
+    static auto SetGlobalLoggingLevel( level::level_enum level )
+    {
+      auto oldval = global_level;
+      global_level = level;
+      return oldval;
+    }
 
     std::shared_ptr<spdlog::logger> logger;
 

libsrc/core/memtracer.hpp

@@ -35,11 +35,16 @@ namespace ngcore
 
   class MemoryTracer
   {
-    #ifdef NETGEN_TRACE_MEMORY
+#if defined(NETGEN_TRACE_MEMORY) && !defined(__CUDA_ARCH__)
     NGCORE_API static std::vector<std::string> names;
     NGCORE_API static std::vector<int> parents;
 
-    static int CreateId(const std::string& name)
+    #if defined(NETGEN_CHECK_RANGE)
+    NGCORE_API static std::atomic<size_t> total_memory;
+    mutable size_t allocated_memory = 0;
+    #endif // NETGEN_CHECK_RANGE
+
+    static int CreateId(const std::string& name = "")
     {
       int id = names.size();
       names.push_back(name);
@@ -48,7 +53,7 @@ namespace ngcore
         std::cerr << "Allocated " << id << " MemoryTracer objects" << std::endl;
       return id;
     }
-    int id;
+    mutable int id = 0;
 
     public:
 
@@ -57,8 +62,33 @@ namespace ngcore
       id = CreateId(name);
     }
 
-    // not tracing
+    MemoryTracer() { }
-    MemoryTracer() : id(0) {}
+
+    MemoryTracer(const MemoryTracer & tracer)
+    {
+      (*this) = tracer;
+    }
+
+    MemoryTracer(MemoryTracer && tracer)
+    {
+      (*this) = std::move(tracer);
+    }
+
+    MemoryTracer & operator=(const MemoryTracer & tracer) {
+      if(tracer.id)
+        id = CreateId(names[tracer.id]);
+      return *this;
+    }
+
+    MemoryTracer & operator=(MemoryTracer && tracer) {
+      ngcore::Swap(id, tracer.id);
+
+      #if defined(NETGEN_CHECK_RANGE)
+      ngcore::Swap(allocated_memory, tracer.allocated_memory);
+      #endif // NETGEN_CHECK_RANGE
+
+      return *this;
+    }
 
     template <typename... TRest>
     MemoryTracer( std::string name, TRest & ... rest )
@@ -67,37 +97,47 @@ namespace ngcore
       Track(rest...);
     }
 
+    #if defined(NETGEN_CHECK_RANGE)
+    // check if all memory was freed when object is destroyed
+    ~MemoryTracer()
+    {
+      NETGEN_CHECK_SAME(allocated_memory, 0);
+    }
+    #endif // NETGEN_CHECK_RANGE
+
     NETGEN_INLINE void Alloc(size_t size) const
     {
+      #if defined(NETGEN_CHECK_RANGE)
+      // Trace also nameless Memtracer objects if range checks are active
+      if(!id && size)
+        id = CreateId();
+      #endif // NETGEN_CHECK_RANGE
+
       if(id && trace)
         trace->AllocMemory(id, size);
+
+      #if defined(NETGEN_CHECK_RANGE)
+      if(id)
+      {
+        allocated_memory += size;
+        total_memory += size;
+      }
+      #endif // NETGEN_CHECK_RANGE
     }
 
     void Free(size_t size) const
     {
       if(id && trace)
         trace->FreeMemory(id, size);
-    }
 
-    void Swap(size_t mysize, MemoryTracer& other, size_t other_size) const
+      #if defined(NETGEN_CHECK_RANGE)
+      if(id)
       {
-      if(!trace || (id == 0 && other.id == 0))
+        // check if we have at least size bytes of memory currently allocated (such that allocated_memory doesn't get negative)
-        return;
+        NETGEN_CHECK_RANGE(allocated_memory, static_cast<ptrdiff_t>(size), std::numeric_limits<ptrdiff_t>::max());
-      if(id == 0)
+        allocated_memory -= size;
-        return trace->ChangeMemory(other.id, mysize - other_size);
+        total_memory -= size;
-      if(other.id == 0)
+        #endif // NETGEN_CHECK_RANGE
-        return trace->ChangeMemory(id, other_size - mysize);
-
-      // first decrease memory, otherwise have artificial/wrong high peak memory usage
-      if(mysize<other_size)
-        {
-          trace->ChangeMemory(other.id, mysize-other_size);
-          trace->ChangeMemory(id, other_size-mysize);
-        }
-      else
-        {
-          trace->ChangeMemory(id, other_size-mysize);
-          trace->ChangeMemory(other.id, mysize-other_size);
       }
     }
 
@@ -148,7 +188,15 @@ namespace ngcore
 
     static const std::vector<std::string> & GetNames() { return names; }
     static const std::vector<int> & GetParents() { return parents; }
-#else // NETGEN_TRACE_MEMORY
+    static size_t GetTotalMemory()
+    {
+      #if defined(NETGEN_CHECK_RANGE)
+      return total_memory;
+      #else
+      return 0;
+      #endif // NETGEN_CHECK_RANGE
+    }
+#else // defined(NETGEN_TRACE_MEMORY) && !defined(__CUDA_ARCH__)
   public:
     MemoryTracer() {}
     MemoryTracer( std::string /* name */ ) {}
@@ -157,7 +205,6 @@ namespace ngcore
 
     void Alloc(size_t /* size */) const {}
     void Free(size_t /* size */) const {}
-    void Swap(...) const {}
     int GetId() const { return 0; }
 
     template <typename... TRest>
@@ -166,6 +213,7 @@ namespace ngcore
     static std::string GetName(int /* id */) { return ""; }
     std::string GetName() const { return ""; }
     void SetName(std::string /* name */) const {}
+    static size_t GetTotalMemory() { return 0; }
 #endif // NETGEN_TRACE_MEMORY
   };
 } // namespace ngcore

libsrc/core/mpi4py_pycapi.h

@@ -0,0 +1,245 @@
+/* Author:  Lisandro Dalcin   */
+/* Contact: dalcinl@gmail.com */
+
+#ifndef MPI4PY_PYCAPI_H
+#define MPI4PY_PYCAPI_H
+
+#include <mpi.h>
+#include <Python.h>
+
+#define _mpi4py_declare_pycapi(Type, star) \
+static PyTypeObject *_mpi4py_PyMPI##Type = NULL; \
+static PyObject *(*_mpi4py_PyMPI##Type##_New)(MPI_##Type star) = NULL; \
+static MPI_##Type *(*_mpi4py_PyMPI##Type##_Get)(PyObject *) = NULL;
+
+#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE
+_mpi4py_declare_pycapi(Datatype,)
+#define PyMPIDatatype_Type (*_mpi4py_PyMPIDatatype)
+#define PyMPIDatatype_New _mpi4py_PyMPIDatatype_New
+#define PyMPIDatatype_Get _mpi4py_PyMPIDatatype_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_STATUS
+_mpi4py_declare_pycapi(Status,*)
+#define PyMPIStatus_Type (*_mpi4py_PyMPIStatus)
+#define PyMPIStatus_New _mpi4py_PyMPIStatus_New
+#define PyMPIStatus_Get _mpi4py_PyMPIStatus_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST
+_mpi4py_declare_pycapi(Request,)
+#define PyMPIRequest_Type (*_mpi4py_PyMPIRequest)
+#define PyMPIRequest_New _mpi4py_PyMPIRequest_New
+#define PyMPIRequest_Get _mpi4py_PyMPIRequest_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE
+_mpi4py_declare_pycapi(Message,)
+#define PyMPIMessage_Type (*_mpi4py_PyMPIMessage)
+#define PyMPIMessage_New _mpi4py_PyMPIMessage_New
+#define PyMPIMessage_Get _mpi4py_PyMPIMessage_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_OP
+_mpi4py_declare_pycapi(Op,)
+#define PyMPIOp_Type (*_mpi4py_PyMPIOp)
+#define PyMPIOp_New _mpi4py_PyMPIOp_New
+#define PyMPIOp_Get _mpi4py_PyMPIOp_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_GROUP
+_mpi4py_declare_pycapi(Group,)
+#define PyMPIGroup_Type (*_mpi4py_PyMPIGroup)
+#define PyMPIGroup_New _mpi4py_PyMPIGroup_New
+#define PyMPIGroup_Get _mpi4py_PyMPIGroup_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_INFO
+_mpi4py_declare_pycapi(Info,)
+#define PyMPIInfo_Type (*_mpi4py_PyMPIInfo)
+#define PyMPIInfo_New _mpi4py_PyMPIInfo_New
+#define PyMPIInfo_Get _mpi4py_PyMPIInfo_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER
+_mpi4py_declare_pycapi(Errhandler,)
+#define PyMPIErrhandler_Type (*_mpi4py_PyMPIErrhandler)
+#define PyMPIErrhandler_New _mpi4py_PyMPIErrhandler_New
+#define PyMPIErrhandler_Get _mpi4py_PyMPIErrhandler_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_SESSION
+_mpi4py_declare_pycapi(Session,)
+#define PyMPISession_Type (*_mpi4py_PyMPISession)
+#define PyMPISession_New _mpi4py_PyMPISession_New
+#define PyMPISession_Get _mpi4py_PyMPISession_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_COMM
+_mpi4py_declare_pycapi(Comm,)
+#define PyMPIComm_Type (*_mpi4py_PyMPIComm)
+#define PyMPIComm_New _mpi4py_PyMPIComm_New
+#define PyMPIComm_Get _mpi4py_PyMPIComm_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_WIN
+_mpi4py_declare_pycapi(Win,)
+#define PyMPIWin_Type (*_mpi4py_PyMPIWin)
+#define PyMPIWin_New _mpi4py_PyMPIWin_New
+#define PyMPIWin_Get _mpi4py_PyMPIWin_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_FILE
+_mpi4py_declare_pycapi(File,)
+#define PyMPIFile_Type (*_mpi4py_PyMPIFile)
+#define PyMPIFile_New _mpi4py_PyMPIFile_New
+#define PyMPIFile_Get _mpi4py_PyMPIFile_Get
+#endif
+
+#undef _mpi4py_define_pycapi
+
+static int _mpi4py_ImportType(PyObject *module,
+                              const char *type_name,
+                              PyTypeObject **type)
+{
+  PyObject *attr = NULL;
+  attr = PyObject_GetAttrString(module, type_name);
+  if (!attr)
+    goto fn_fail;
+  if (!PyType_Check(attr)) {
+    PyErr_Format(PyExc_TypeError,
+                 "%.200s.%.200s is not a type object",
+                 PyModule_GetName(module), type_name);
+    goto fn_fail;
+  }
+  *type = (PyTypeObject *)attr;
+  return 0;
+ fn_fail:
+  Py_DecRef(attr);
+  return -1;
+}
+
+static int _mpi4py_ImportFunc(PyObject *module,
+                              const char *func_name,
+                              const char *signature,
+                              void (**func)(void))
+{
+    PyObject *pyxcapi = NULL;
+    PyObject *capsule = NULL;
+    union { void *obj; void (*fcn)(void); } ptr;
+    pyxcapi = PyObject_GetAttrString(module, (char *)"__pyx_capi__");
+    if (!pyxcapi)
+      goto fn_fail;
+
+    capsule = PyDict_GetItemString(pyxcapi, func_name);
+    if (!capsule) {
+      PyErr_Format(PyExc_ImportError,
+                   "%.200s does not export expected C function %.200s",
+                   PyModule_GetName(module), func_name);
+      goto fn_fail;
+    }
+    if (!PyCapsule_CheckExact(capsule)) {
+      PyErr_Format(PyExc_TypeError,
+                   "%.200s.%.200s is not a capsule",
+                   PyModule_GetName(module), func_name);
+    }
+    if (!signature) {
+      signature = PyCapsule_GetName(capsule);
+    }
+    if (!PyCapsule_IsValid(capsule, signature)) {
+      PyErr_Format(PyExc_TypeError,
+                   "C function %.200s.%.200s has wrong signature "
+                   "(expected %.500s, got %.500s)",
+                   PyModule_GetName(module), func_name,
+                   signature, PyCapsule_GetName(capsule));
+      goto fn_fail;
+    }
+    ptr.obj = PyCapsule_GetPointer(capsule, signature);
+    if (!ptr.obj)
+      goto fn_fail;
+    *func = ptr.fcn;
+    Py_DecRef(pyxcapi);
+    return 0;
+ fn_fail:
+    Py_DecRef(pyxcapi);
+    return -1;
+}
+
+static int import_mpi4py_MPI(void)
+{
+  PyObject *module = PyImport_ImportModule("mpi4py.MPI");
+  if (!module)
+    goto fn_fail;
+
+#define _mpi4py_import_pycapi(Type) do {                                     \
+    if (_mpi4py_ImportType(module, #Type, &_mpi4py_PyMPI##Type) < 0)         \
+      goto fn_fail;                                                          \
+    if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_New", NULL,               \
+                           (void (**)(void))&_mpi4py_PyMPI##Type##_New) < 0) \
+      goto fn_fail;                                                          \
+    if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_Get", NULL,               \
+                           (void (**)(void))&_mpi4py_PyMPI##Type##_Get) < 0) \
+      goto fn_fail;                                                          \
+  } while (0)
+
+#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE
+  _mpi4py_import_pycapi(Datatype);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_STATUS
+  _mpi4py_import_pycapi(Status);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST
+  _mpi4py_import_pycapi(Request);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE
+  _mpi4py_import_pycapi(Message);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_OP
+  _mpi4py_import_pycapi(Op);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_GROUP
+  _mpi4py_import_pycapi(Group);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_INFO
+  _mpi4py_import_pycapi(Info);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER
+  _mpi4py_import_pycapi(Errhandler);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_SESSION
+  _mpi4py_import_pycapi(Session);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_COMM
+  _mpi4py_import_pycapi(Comm);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_WIN
+  _mpi4py_import_pycapi(Win);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_FILE
+  _mpi4py_import_pycapi(File);
+#endif
+
+#undef _mpi4py_import_pycapi
+
+  Py_DecRef(module);
+  return 0;
+ fn_fail:
+  Py_DecRef(module);
+  return -1;
+}
+
+#define __PYX_HAVE_API__mpi4py__MPI
+#define import_mpi4py__MPI import_mpi4py_MPI
+
+#endif /* MPI4PY_PYCAPI_H */

libsrc/core/mpi_wrapper.hpp

@@ -3,16 +3,14 @@
 
 #include <array>
 
-#ifdef PARALLEL
+#include <complex>
-#define OMPI_SKIP_MPICXX
-#include <mpi.h>
-#endif
 
 #include "array.hpp"
 #include "table.hpp"
 #include "exception.hpp"
 #include "profiler.hpp"
 #include "ngstream.hpp"
+#include "ng_mpi.hpp"
 
 namespace ngcore
 {
@@ -22,67 +20,124 @@ namespace ngcore
   template <class T> struct MPI_typetrait  { };
   
   template <> struct MPI_typetrait<int> {
-    static MPI_Datatype MPIType () { return MPI_INT; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_INT; } };
 
   template <> struct MPI_typetrait<short> {
-    static MPI_Datatype MPIType () { return MPI_SHORT; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_SHORT; } };
 
   template <> struct MPI_typetrait<char> {
-    static MPI_Datatype MPIType () { return MPI_CHAR; } };  
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
 
   template <> struct MPI_typetrait<signed char> {
-    static MPI_Datatype MPIType () { return MPI_CHAR; } };  
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
   
   template <> struct MPI_typetrait<unsigned char> {
-    static MPI_Datatype MPIType () { return MPI_CHAR; } };  
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
 
   template <> struct MPI_typetrait<size_t> {
-    static MPI_Datatype MPIType () { return MPI_UINT64_T; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_UINT64_T; } };
 
   template <> struct MPI_typetrait<double> {
-    static MPI_Datatype MPIType () { return MPI_DOUBLE; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_DOUBLE; } };
+
+  template <> struct MPI_typetrait<std::complex<double>> {
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CXX_DOUBLE_COMPLEX; } };
 
   template <> struct MPI_typetrait<bool> {
-    static MPI_Datatype MPIType () { return MPI_C_BOOL; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_C_BOOL; } };
 
 
   template<typename T, size_t S>
   struct MPI_typetrait<std::array<T,S>>
   {
-    static MPI_Datatype MPIType () 
+    static NG_MPI_Datatype MPIType ()
     { 
-      static MPI_Datatype MPI_T = 0;
+      static NG_MPI_Datatype NG_MPI_T = 0;
-      if (!MPI_T)
+      if (!NG_MPI_T)
 	{
-	  MPI_Type_contiguous ( S, MPI_typetrait<T>::MPIType(), &MPI_T);
+	  NG_MPI_Type_contiguous ( S, MPI_typetrait<T>::MPIType(), &NG_MPI_T);
-	  MPI_Type_commit ( &MPI_T );
+	  NG_MPI_Type_commit ( &NG_MPI_T );
 	}
-      return MPI_T;
+      return NG_MPI_T;
     }
   };
   
   template <class T, class T2 = decltype(MPI_typetrait<T>::MPIType())>
-  inline MPI_Datatype GetMPIType () {
+  inline NG_MPI_Datatype GetMPIType () {
     return MPI_typetrait<T>::MPIType();
   }
 
   template <class T>
-  inline MPI_Datatype GetMPIType (T &) {
+  inline NG_MPI_Datatype GetMPIType (T &) {
     return GetMPIType<T>();
   }
 
+  class NgMPI_Request
+  {
+    NG_MPI_Request request;
+  public:
+    NgMPI_Request (NG_MPI_Request requ) : request{requ} { }
+    NgMPI_Request (const NgMPI_Request&) = delete;    
+    NgMPI_Request (NgMPI_Request&&) = default;
+    ~NgMPI_Request () { NG_MPI_Wait (&request, NG_MPI_STATUS_IGNORE); }
+    void Wait() {  NG_MPI_Wait (&request, NG_MPI_STATUS_IGNORE); }
+    operator NG_MPI_Request() &&
+    {
+      auto tmp = request;
+      request = NG_MPI_REQUEST_NULL;
+      return tmp;
+    }
+  };
 
-  inline void MyMPI_WaitAll (FlatArray<MPI_Request> requests)
+  class NgMPI_Requests
+  {
+    Array<NG_MPI_Request> requests;
+  public:
+    NgMPI_Requests() = default;
+    ~NgMPI_Requests() { WaitAll(); }
+
+    void Reset() { requests.SetSize0(); }
+    
+    NgMPI_Requests & operator+= (NgMPI_Request && r)
+    {
+      requests += NG_MPI_Request(std::move(r));
+      return *this;
+    }
+
+    NgMPI_Requests & operator+= (NG_MPI_Request r)
+    {
+      requests += r;
+      return *this;
+    }
+    
+    void WaitAll()
+    {
+      static Timer t("NgMPI - WaitAll"); RegionTimer reg(t);    
+      if (!requests.Size()) return;
+      NG_MPI_Waitall (requests.Size(), requests.Data(), NG_MPI_STATUSES_IGNORE);
+    }
+
+    int WaitAny ()
+    {
+      int nr;
+      NG_MPI_Waitany (requests.Size(), requests.Data(), &nr, NG_MPI_STATUS_IGNORE);
+      return nr;
+    }
+  };
+  
+  [[deprecated("use requests.WaitAll instread")]]
+  inline void MyMPI_WaitAll (FlatArray<NG_MPI_Request> requests)
   {
     static Timer t("MPI - WaitAll"); RegionTimer reg(t);    
     if (!requests.Size()) return;
-    MPI_Waitall (requests.Size(), requests.Data(), MPI_STATUSES_IGNORE);
+    NG_MPI_Waitall (requests.Size(), requests.Data(), NG_MPI_STATUSES_IGNORE);
   }
 
-  inline int MyMPI_WaitAny (FlatArray<MPI_Request> requests)
+  [[deprecated("use requests.WaitAny instread")]]  
+  inline int MyMPI_WaitAny (FlatArray<NG_MPI_Request> requests)
   {
     int nr;
-    MPI_Waitany (requests.Size(), requests.Data(), &nr, MPI_STATUS_IGNORE);
+    NG_MPI_Waitany (requests.Size(), requests.Data(), &nr, NG_MPI_STATUS_IGNORE);
     return nr;
   }
 
@@ -91,7 +146,7 @@ namespace ngcore
   class NgMPI_Comm
   {
   protected:
-    MPI_Comm comm;
+    NG_MPI_Comm comm;
     bool valid_comm;
     int * refcount;
     int rank, size;
@@ -100,11 +155,11 @@ namespace ngcore
       : valid_comm(false), refcount(nullptr), rank(0), size(1)
     { ; }
 
-    NgMPI_Comm (MPI_Comm _comm, bool owns = false)
+    NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false)
       : comm(_comm), valid_comm(true)
     {
       int flag;
-      MPI_Initialized (&flag);
+      NG_MPI_Initialized (&flag);
       if (!flag)
         {
           valid_comm = false;
@@ -119,8 +174,8 @@ namespace ngcore
       else
         refcount = new int{1};
       
-      MPI_Comm_rank(comm, &rank);
+      NG_MPI_Comm_rank(comm, &rank);
-      MPI_Comm_size(comm, &size);
+      NG_MPI_Comm_size(comm, &size);
     }
     
     NgMPI_Comm (const NgMPI_Comm & c)
@@ -141,7 +196,7 @@ namespace ngcore
     {
       if (refcount)
         if (--(*refcount) == 0)
-          MPI_Comm_free(&comm);
+          NG_MPI_Comm_free(&comm);
     }
 
     bool ValidCommunicator() const
@@ -153,7 +208,7 @@ namespace ngcore
     {
       if (refcount)
         if (--(*refcount) == 0)
-          MPI_Comm_free(&comm);
+          NG_MPI_Comm_free(&comm);
 
       refcount = c.refcount;
       if (refcount) (*refcount)++;      
@@ -169,7 +224,7 @@ namespace ngcore
       InvalidCommException() : Exception("Do not have a valid communicator") { ; }
     };
     
-    operator MPI_Comm() const {
+    operator NG_MPI_Comm() const {
       if (!valid_comm) throw InvalidCommException();
       return comm;
     }
@@ -178,7 +233,7 @@ namespace ngcore
     int Size() const { return size; }
     void Barrier() const {
       static Timer t("MPI - Barrier"); RegionTimer reg(t);
-      if (size > 1) MPI_Barrier (comm);
+      if (size > 1) NG_MPI_Barrier (comm);
     }
     
 
@@ -186,82 +241,82 @@ namespace ngcore
 
     template<typename T, typename T2 = decltype(GetMPIType<T>())>
     void Send (T & val, int dest, int tag) const {
-      MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
+      NG_MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
     }
 
     void Send (const std::string & s, int dest, int tag) const {
-      MPI_Send( const_cast<char*> (&s[0]), s.length(), MPI_CHAR, dest, tag, comm);
+      NG_MPI_Send( const_cast<char*> (&s[0]), s.length(), NG_MPI_CHAR, dest, tag, comm);
     }
     
     template<typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
     void Send(FlatArray<T,TI> s, int dest, int tag) const {
-      MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
+      NG_MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
     void Recv (T & val, int src, int tag) const {
-      MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, MPI_STATUS_IGNORE);
+      NG_MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, NG_MPI_STATUS_IGNORE);
     }
 
     void Recv (std::string & s, int src, int tag) const {    
-      MPI_Status status;
+      NG_MPI_Status status;
       int len;
-      MPI_Probe (src, tag, comm, &status);
+      NG_MPI_Probe (src, tag, comm, &status);
-      MPI_Get_count (&status, MPI_CHAR, &len);
+      NG_MPI_Get_count (&status, NG_MPI_CHAR, &len);
       // s.assign (len, ' ');
       s.resize (len);
-      MPI_Recv( &s[0], len, MPI_CHAR, src, tag, comm, MPI_STATUS_IGNORE);
+      NG_MPI_Recv( &s[0], len, NG_MPI_CHAR, src, tag, comm, NG_MPI_STATUS_IGNORE);
     }
     
 
     template <typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
     void Recv (FlatArray <T,TI> s, int src, int tag) const {
-      MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, MPI_STATUS_IGNORE);
+      NG_MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, NG_MPI_STATUS_IGNORE);
     }
     
     template <typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
     void Recv (Array <T,TI> & s, int src, int tag) const
     {
-      MPI_Status status;
+      NG_MPI_Status status;
       int len;
-      const MPI_Datatype MPI_T  = GetMPIType<T> ();
+      const NG_MPI_Datatype NG_MPI_T  = GetMPIType<T> ();
-      MPI_Probe (src, tag, comm, &status);
+      NG_MPI_Probe (src, tag, comm, &status);
-      MPI_Get_count (&status, MPI_T, &len);
+      NG_MPI_Get_count (&status, NG_MPI_T, &len);
       s.SetSize (len);
-      MPI_Recv (s.Data(), len, MPI_T, src, tag, comm, MPI_STATUS_IGNORE);
+      NG_MPI_Recv (s.Data(), len, NG_MPI_T, src, tag, comm, NG_MPI_STATUS_IGNORE);
     }
 
     /** --- non-blocking P2P --- **/
 
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
-    MPI_Request ISend (T & val, int dest, int tag) const
+    [[nodiscard]] NG_MPI_Request ISend (T & val, int dest, int tag) const
     {
-      MPI_Request request;
+      NG_MPI_Request request;
-      MPI_Isend (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
+      NG_MPI_Isend (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
       return request;
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())>
-    MPI_Request ISend (FlatArray<T> s, int dest, int tag) const
+    [[nodiscard]] NG_MPI_Request ISend (FlatArray<T> s, int dest, int tag) const
     {
-      MPI_Request request;
+      NG_MPI_Request request;
-      MPI_Isend (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
+      NG_MPI_Isend (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
       return request;
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
-    MPI_Request IRecv (T & val, int dest, int tag) const
+    [[nodiscard]] NG_MPI_Request IRecv (T & val, int dest, int tag) const
     {
-      MPI_Request request;
+      NG_MPI_Request request;
-      MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
+      NG_MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
       return request;
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())>
-    MPI_Request IRecv (FlatArray<T> s, int src, int tag) const
+    [[nodiscard]] NG_MPI_Request IRecv (FlatArray<T> s, int src, int tag) const
     { 
-      MPI_Request request;
+      NG_MPI_Request request;
-      MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
+      NG_MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
       return request;
     }
 
@@ -269,46 +324,55 @@ namespace ngcore
     /** --- collectives --- **/
 
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
-    T Reduce (T d, const MPI_Op & op, int root = 0) const
+    T Reduce (T d, const NG_MPI_Op & op, int root = 0) const
     {
       static Timer t("MPI - Reduce"); RegionTimer reg(t);          
       if (size == 1) return d;
       
       T global_d;
-      MPI_Reduce (&d, &global_d, 1, GetMPIType<T>(), op, root, comm);
+      NG_MPI_Reduce (&d, &global_d, 1, GetMPIType<T>(), op, root, comm);
       return global_d;
     }
     
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
-    T AllReduce (T d, const MPI_Op & op) const
+    T AllReduce (T d, const NG_MPI_Op & op) const
     {
       static Timer t("MPI - AllReduce"); RegionTimer reg(t);
       if (size == 1) return d;
       
       T global_d;
-      MPI_Allreduce ( &d, &global_d, 1, GetMPIType<T>(), op, comm);
+      NG_MPI_Allreduce ( &d, &global_d, 1, GetMPIType<T>(), op, comm);
       return global_d;
     }
 
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
-    void AllReduce (FlatArray<T> d, const MPI_Op & op) const
+    void AllReduce (FlatArray<T> d, const NG_MPI_Op & op) const
     {
       static Timer t("MPI - AllReduce Array"); RegionTimer reg(t);
       if (size == 1) return;
       
-      MPI_Allreduce (MPI_IN_PLACE, d.Data(), d.Size(), GetMPIType<T>(), op, comm);
+      NG_MPI_Allreduce (NG_MPI_IN_PLACE, d.Data(), d.Size(), GetMPIType<T>(), op, comm);
     }
     
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
     void Bcast (T & s, int root = 0) const {
       if (size == 1) return;
       static Timer t("MPI - Bcast"); RegionTimer reg(t);
-      MPI_Bcast (&s, 1, GetMPIType<T>(), root, comm);
+      NG_MPI_Bcast (&s, 1, GetMPIType<T>(), root, comm);
+    }
+
+
+    template <class T, size_t S>
+    void Bcast (std::array<T,S> & d, int root = 0) const
+    {
+      if (size == 1) return;
+      if (S != 0)
+        NG_MPI_Bcast (&d[0], S, GetMPIType<T>(), root, comm);
     }
     
     
     template <class T>
-    void Bcast (Array<T> & d, int root = 0)
+    void Bcast (Array<T> & d, int root = 0) const
     {
       if (size == 1) return;
       
@@ -316,7 +380,7 @@ namespace ngcore
       Bcast (ds, root);
       if (Rank() != root) d.SetSize (ds);
       if (ds != 0)
-        MPI_Bcast (d.Data(), ds, GetMPIType<T>(), root, comm);
+        NG_MPI_Bcast (d.Data(), ds, GetMPIType<T>(), root, comm);
     }
 
     
@@ -326,13 +390,33 @@ namespace ngcore
       int len = s.length();
       Bcast (len, root);
       if (rank != 0) s.resize (len);
-      MPI_Bcast (&s[0], len, MPI_CHAR, root, comm);
+      NG_MPI_Bcast (&s[0], len, NG_MPI_CHAR, root, comm);
     }
 
+
+    
+    template <class T, size_t S>
+    [[nodiscard]] NgMPI_Request IBcast (std::array<T,S> & d, int root = 0) const
+    {
+      NG_MPI_Request request;      
+      NG_MPI_Ibcast (&d[0], S, GetMPIType<T>(), root, comm, &request);
+      return request;
+    }
+
+    template <class T>
+     [[nodiscard]] NgMPI_Request IBcast (FlatArray<T> d, int root = 0) const
+    {
+      NG_MPI_Request request;      
+      int ds = d.Size();
+      NG_MPI_Ibcast (d.Data(), ds, GetMPIType<T>(), root, comm, &request);
+      return request;
+    }
+
+    
     template <typename T>
     void AllToAll (FlatArray<T> send, FlatArray<T> recv) const
     {
-      MPI_Alltoall (send.Data(), 1, GetMPIType<T>(), 
+      NG_MPI_Alltoall (send.Data(), 1, GetMPIType<T>(),
                        recv.Data(), 1, GetMPIType<T>(), comm);
     }
 
@@ -341,15 +425,15 @@ namespace ngcore
     void ScatterRoot (FlatArray<T> send) const
     {
       if (size == 1) return;
-      MPI_Scatter (send.Data(), 1, GetMPIType<T>(),
+      NG_MPI_Scatter (send.Data(), 1, GetMPIType<T>(),
-                   MPI_IN_PLACE, -1, GetMPIType<T>(), 0, comm);
+                      NG_MPI_IN_PLACE, -1, GetMPIType<T>(), 0, comm);
     }
     
     template <typename T>
     void Scatter (T & recv) const
     {
       if (size == 1) return;      
-      MPI_Scatter (NULL, 0, GetMPIType<T>(),
+      NG_MPI_Scatter (NULL, 0, GetMPIType<T>(),
                       &recv, 1, GetMPIType<T>(), 0, comm);
     }
 
@@ -358,7 +442,7 @@ namespace ngcore
     {
       recv[0] = T(0);
       if (size == 1) return;      
-      MPI_Gather (MPI_IN_PLACE, 1, GetMPIType<T>(), 
+      NG_MPI_Gather (NG_MPI_IN_PLACE, 1, GetMPIType<T>(),
                      recv.Data(), 1, GetMPIType<T>(), 0, comm);
     }
 
@@ -366,7 +450,7 @@ namespace ngcore
     void Gather (T send) const
     {
       if (size == 1) return;            
-      MPI_Gather (&send, 1, GetMPIType<T>(), 
+      NG_MPI_Gather (&send, 1, GetMPIType<T>(),
                   NULL, 1, GetMPIType<T>(), 0, comm);
     }
 
@@ -379,7 +463,7 @@ namespace ngcore
           recv[0] = val;
           return;
         }
-      MPI_Allgather (&val, 1, GetMPIType<T>(),
+      NG_MPI_Allgather (&val, 1, GetMPIType<T>(),
                      recv.Data(), 1, GetMPIType<T>(), 
                      comm);
     }
@@ -400,16 +484,16 @@ namespace ngcore
     
       recv_data = DynamicTable<T> (recv_sizes, true);
       
-      Array<MPI_Request> requests;
+      NgMPI_Requests requests;
       for (int dest = 0; dest < size; dest++)
         if (dest != rank && send_data[dest].Size())
-          requests.Append (ISend (FlatArray<T>(send_data[dest]), dest, tag));
+          requests += ISend (FlatArray<T>(send_data[dest]), dest, tag);
       
       for (int dest = 0; dest < size; dest++)
         if (dest != rank && recv_data[dest].Size())
-          requests.Append (IRecv (FlatArray<T>(recv_data[dest]), dest, tag));
+          requests += IRecv (FlatArray<T>(recv_data[dest]), dest, tag);
 
-      MyMPI_WaitAll (requests);
+      requests.WaitAll();
     }
     
 
@@ -418,92 +502,69 @@ namespace ngcore
     
     NgMPI_Comm SubCommunicator (FlatArray<int> procs) const
     {
-      MPI_Comm subcomm;
+      NG_MPI_Comm subcomm;
-      MPI_Group gcomm, gsubcomm;
+      NG_MPI_Group gcomm, gsubcomm;
-      MPI_Comm_group(comm, &gcomm);
+      NG_MPI_Comm_group(comm, &gcomm);
-      MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm);
+      NG_MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm);
-      MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm);
+      NG_MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm);
       return NgMPI_Comm(subcomm, true);
     }
 
   }; // class NgMPI_Comm
 
-
-
-
-
-  
-  class MyMPI
-  {
-    bool initialized_by_me;
-  public:
-    MyMPI(int argc, char ** argv) 
-    {
-      int is_init = -1;
-      MPI_Initialized(&is_init);
-      if (!is_init)
-        {
-          MPI_Init (&argc, &argv);
-          initialized_by_me = true;
-        }
-      else
-        initialized_by_me = false;
-      
-      NgMPI_Comm comm(MPI_COMM_WORLD);
-      NGSOStream::SetGlobalActive (comm.Rank() == 0);
-      
-      if (comm.Size() > 1)
-        TaskManager::SetNumThreads (1);
-    }
-    
-    ~MyMPI()
-    {
-      if (initialized_by_me)
-        MPI_Finalize ();
-    }
-  };
-  
-
-
-  
-
 #else // PARALLEL
-  class MPI_Comm {
+  class NG_MPI_Comm {
     int nr;
   public:
-    MPI_Comm (int _nr = 0) : nr(_nr) { ; }
+    NG_MPI_Comm (int _nr = 0) : nr(_nr) { ; }
     operator int() const { return nr; }
-    bool operator== (MPI_Comm c2) const { return nr == c2.nr; }
+    bool operator== (NG_MPI_Comm c2) const { return nr == c2.nr; }
   };
-  static MPI_Comm MPI_COMM_WORLD = 12345, MPI_COMM_NULL = 10000;
+  static NG_MPI_Comm NG_MPI_COMM_WORLD = 12345, NG_MPI_COMM_NULL = 10000;
 
-  typedef int MPI_Op;
+  typedef int NG_MPI_Op;
-  typedef int MPI_Datatype;  
+  typedef int NG_MPI_Datatype;  
-  typedef int MPI_Request;
+  typedef int NG_MPI_Request;
   
-  enum { MPI_SUM = 0, MPI_MIN = 1, MPI_MAX = 2, MPI_LOR = 4711 };
+  enum { NG_MPI_SUM = 0, NG_MPI_MIN = 1, NG_MPI_MAX = 2, NG_MPI_LOR = 4711 };
 
-  inline void MPI_Type_contiguous ( int, MPI_Datatype, MPI_Datatype*) { ; } 
+  inline void NG_MPI_Type_contiguous ( int, NG_MPI_Datatype, NG_MPI_Datatype*) { ; } 
-  inline void MPI_Type_commit ( MPI_Datatype * ) { ; }
+  inline void NG_MPI_Type_commit ( NG_MPI_Datatype * ) { ; }
 
   template <class T> struct MPI_typetrait  {
-    static MPI_Datatype MPIType () { return -1; }    
+    static NG_MPI_Datatype MPIType () { return -1; }    
   };
   template <class T, class T2=void>
-  inline MPI_Datatype GetMPIType () { return -1; }
+  inline NG_MPI_Datatype GetMPIType () { return -1; }
+
+  class NgMPI_Request {
+  public:
+    NgMPI_Request() = default;
+    NgMPI_Request(NgMPI_Request &&) { ; }    
+    NgMPI_Request(NG_MPI_Request &&) { ; }
+  };
+  class NgMPI_Requests
+  {
+  public:
+    NgMPI_Requests & operator+= (NgMPI_Request &&) { return *this; }
+    NgMPI_Requests & operator+= (NG_MPI_Request r) { return *this; }
+    void Reset() { ; }
+    void WaitAll() { ; }
+    int WaitAny() { return 0; }
+  };
   
   class NgMPI_Comm
   {
     
   public:
     NgMPI_Comm () { ; } 
-    NgMPI_Comm (MPI_Comm _comm, bool owns = false) { ; }
+    NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false) { ; }
 
     size_t Rank() const { return 0; }
     size_t Size() const { return 1; }
     bool ValidCommunicator() const { return false; }
     void Barrier() const { ; } 
-    operator MPI_Comm() const { return MPI_Comm(); }
+    operator NG_MPI_Comm() const { return NG_MPI_Comm(); }
 
     template<typename T>
     void Send( T & val, int dest, int tag) const { ; }
@@ -521,31 +582,40 @@ namespace ngcore
     void Recv (Array <T> & s, int src, int tag) const { ; }
 
     template<typename T>
-    MPI_Request ISend (T & val, int dest, int tag) const { return 0; } 
+    NG_MPI_Request ISend (T & val, int dest, int tag) const { return 0; } 
     
     template<typename T>
-    MPI_Request ISend (FlatArray<T> s, int dest, int tag) const { return 0; }
+    NG_MPI_Request ISend (FlatArray<T> s, int dest, int tag) const { return 0; }
 
     template<typename T>
-    MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } 
+    NG_MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } 
     
     template<typename T>
-    MPI_Request IRecv (FlatArray<T> s, int src, int tag) const { return 0; }
+    NG_MPI_Request IRecv (FlatArray<T> s, int src, int tag) const { return 0; }
 
     template <typename T>
-    T Reduce (T d, const MPI_Op & op, int root = 0) const { return d; }
+    T Reduce (T d, const NG_MPI_Op & op, int root = 0) const { return d; }
     
     template <typename T>
-    T AllReduce (T d, const MPI_Op & op) const { return d; }
+    T AllReduce (T d, const NG_MPI_Op & op) const { return d; }
 
     template <typename T>
-    void AllReduce (FlatArray<T> d, const MPI_Op & op) const { ; }
+    void AllReduce (FlatArray<T> d, const NG_MPI_Op & op) const { ; }
     
     template <typename T>
     void Bcast (T & s, int root = 0) const { ; } 
 
+    template <class T, size_t S>
+    void Bcast (std::array<T,S> & d, int root = 0) const {}
+    
     template <class T>
-    void Bcast (Array<T> & d, int root = 0) { ; } 
+    void Bcast (Array<T> & d, int root = 0) const { ; } 
+
+    template <class T, size_t S>
+    NG_MPI_Request IBcast (std::array<T,S> & d, int root = 0) const { return 0; }
+
+    template <class T>
+    NG_MPI_Request IBcast (FlatArray<T> d, int root = 0) const { return 0; } 
     
     template <typename T>
     void AllGather (T val, FlatArray<T> recv) const
@@ -562,15 +632,8 @@ namespace ngcore
     { return *this; }
   };  
 
-  inline void MyMPI_WaitAll (FlatArray<MPI_Request> requests) { ; }
+  inline void MyMPI_WaitAll (FlatArray<NG_MPI_Request> requests) { ; }
-  inline int MyMPI_WaitAny (FlatArray<MPI_Request> requests) { return 0; }
+  inline int MyMPI_WaitAny (FlatArray<NG_MPI_Request> requests) { return 0; }
-
-  class MyMPI
-  {
-  public:
-    MyMPI(int argc, char ** argv) { ; }
-  };
-
 
 #endif // PARALLEL
 

libsrc/core/ng_mpi.cpp

@@ -0,0 +1,184 @@
+#define OMPI_SKIP_MPICXX
+#include <mpi.h>
+
+#include "ng_mpi.hpp"
+
+#include <type_traits>
+
+#include "array.hpp"
+#include "ngcore_api.hpp"
+
+#ifdef NG_PYTHON
+#include "pybind11/pytypes.h"
+#include "python_ngcore.hpp"
+
+#define MPI4PY_LIMITED_API 1
+#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1
+#define MPI4PY_LIMITED_API_SKIP_SESSION 1
+#include "mpi4py_pycapi.h"  // mpi4py < 4.0.0
+#endif
+
+#ifdef MSMPI_VER
+int MPI_Comm_create_group(MPI_Comm arg0, MPI_Group arg1, int arg2,
+                          MPI_Comm* arg3) {
+  throw std::runtime_error(
+      "MPI_Comm_create_group not supported on Microsoft MPI");
+}
+static MPI_Datatype MPI_CXX_DOUBLE_COMPLEX;
+#endif  // MSMPI_VER
+
+namespace ngcore {
+
+static_assert(sizeof(MPI_Status) <= sizeof(NG_MPI_Status), "Size mismatch");
+static_assert(alignof(MPI_Status) <= alignof(NG_MPI_Status), "Size mismatch");
+
+int mpi2ng(int value) { return value; }
+void* mpi2ng(void* ptr) { return ptr; }
+
+NG_MPI_Status* mpi2ng(MPI_Status* status) {
+  return reinterpret_cast<NG_MPI_Status*>(status);
+}
+
+#if !defined(MPICH) && !defined(MSMPI_VER)
+NG_MPI_Comm mpi2ng(MPI_Comm comm) { return reinterpret_cast<uintptr_t>(comm); }
+#endif
+
+template <size_t size, size_t stride>
+void gather_strided_array(size_t count, char* data) {
+  static_assert(size <= stride, "Size must be less than or equal to stride");
+  if constexpr (size < stride) {
+    char* dst = data;
+    char* src = data;
+    for ( [[maybe_unused]] auto i : Range(count)) {
+      memcpy(dst, src, size);
+      dst += size;
+      src += stride;
+    }
+  }
+}
+
+template <typename T>
+T cast_ng2mpi(uintptr_t obj) {
+  if constexpr (std::is_pointer_v<T>)
+    return reinterpret_cast<T>(obj);
+  else
+    return static_cast<T>(obj);
+}
+
+template <typename T>
+T cast_ng2mpi(uintptr_t* ptr) {
+  if constexpr (std::is_pointer_v<T>)
+    return reinterpret_cast<T>(ptr);
+  else
+    return static_cast<T>(ptr);
+}
+
+template <typename T, typename TSrc>
+T* cast_ng2mpi(TSrc* ptr, int count) {
+  gather_strided_array<sizeof(T), sizeof(TSrc)>(count,
+                                                reinterpret_cast<char*>(ptr));
+  return reinterpret_cast<T*>(ptr);
+}
+
+MPI_Comm ng2mpi(NG_MPI_Comm comm) {
+  static_assert(sizeof(MPI_Comm) <= sizeof(comm.value), "Size mismatch");
+  static_assert(alignof(MPI_Comm) <= alignof(NG_MPI_Comm), "Size mismatch");
+  return cast_ng2mpi<MPI_Comm>(comm.value);
+}
+
+MPI_Group ng2mpi(NG_MPI_Group group) {
+  static_assert(sizeof(MPI_Group) <= sizeof(group.value), "Size mismatch");
+  static_assert(alignof(MPI_Group) <= alignof(NG_MPI_Group), "Size mismatch");
+  return cast_ng2mpi<MPI_Group>(group.value);
+}
+
+MPI_Comm* ng2mpi(NG_MPI_Comm* comm) {
+  return cast_ng2mpi<MPI_Comm*>(&comm->value);
+}
+MPI_Group* ng2mpi(NG_MPI_Group* group) {
+  return cast_ng2mpi<MPI_Group*>(&group->value);
+}
+MPI_Datatype* ng2mpi(NG_MPI_Datatype* type) {
+  return cast_ng2mpi<MPI_Datatype*>(&type->value);
+}
+MPI_Datatype* ng2mpi(NG_MPI_Datatype* type, int count) {
+  return cast_ng2mpi<MPI_Datatype>(&type->value, count);
+}
+MPI_Request* ng2mpi(NG_MPI_Request* request) {
+  return cast_ng2mpi<MPI_Request*>(&request->value);
+}
+MPI_Request* ng2mpi(NG_MPI_Request* request, int count) {
+  return cast_ng2mpi<MPI_Request>(&request->value, count);
+}
+MPI_Status* ng2mpi(NG_MPI_Status* status) {
+  return reinterpret_cast<MPI_Status*>(status);
+}
+MPI_Aint* ng2mpi(NG_MPI_Aint* aint) {
+  return reinterpret_cast<MPI_Aint*>(aint);
+}
+MPI_Aint* ng2mpi(NG_MPI_Aint* aint, int count) {
+  return cast_ng2mpi<MPI_Aint>(aint, count);
+}
+
+MPI_Datatype ng2mpi(NG_MPI_Datatype type) {
+  static_assert(sizeof(MPI_Datatype) <= sizeof(type.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Datatype>(type.value);
+}
+
+MPI_Request ng2mpi(NG_MPI_Request request) {
+  static_assert(sizeof(MPI_Request) <= sizeof(request.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Request>(request.value);
+}
+
+MPI_Op ng2mpi(NG_MPI_Op op) {
+  static_assert(sizeof(MPI_Op) <= sizeof(op.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Op>(op.value);
+}
+
+MPI_Aint ng2mpi(NG_MPI_Aint aint) {
+  static_assert(sizeof(MPI_Aint) <= sizeof(aint.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Aint>(aint.value);
+}
+
+void* ng2mpi(void* ptr) { return ptr; }
+char* ng2mpi(char* ptr) { return ptr; }
+char*** ng2mpi(char*** ptr) { return ptr; }
+int* ng2mpi(int* ptr) { return ptr; }
+int ng2mpi(int value) { return value; }
+
+}  // namespace ngcore
+
+using namespace ngcore;
+
+extern "C" {
+NGCORE_API_EXPORT void ng_init_mpi();
+}
+
+static bool imported_mpi4py = false;
+
+void ng_init_mpi() {
+#ifdef NG_PYTHON
+  NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool {
+    if (!imported_mpi4py) {
+      import_mpi4py__MPI();
+      imported_mpi4py = true;
+    }
+    PyObject* py_src = src.ptr();
+    [[maybe_unused]] auto type = Py_TYPE(py_src);
+    if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
+      dst = mpi2ng(*PyMPIComm_Get(py_src));
+      return !PyErr_Occurred();
+    }
+    return false;
+  };
+  NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle {
+    if (!imported_mpi4py) {
+      import_mpi4py__MPI();
+      imported_mpi4py = true;
+    }
+    return py::handle(PyMPIComm_New(ng2mpi(src)));
+  };
+#endif  // NG_PYTHON
+
+#include "ng_mpi_generated_init.hpp"
+}

libsrc/core/ng_mpi.hpp

@@ -0,0 +1,94 @@
+#ifndef NG_MPI_HPP_INCLUDED
+#define NG_MPI_HPP_INCLUDED
+
+#ifdef PARALLEL
+
+#include <cstdint>
+#include <filesystem>
+#include <optional>
+
+#include "ngcore_api.hpp"
+
+#ifndef NG_MPI_WRAPPER
+#define OMPI_SKIP_MPICXX
+#include <mpi.h>
+#endif  // NG_MPI_WRAPPER
+
+namespace ngcore {
+
+NGCORE_API bool MPI_Loaded();
+NGCORE_API void InitMPI(
+    std::optional<std::filesystem::path> mpi_lib_path = std::nullopt);
+
+#ifdef NG_MPI_WRAPPER
+inline void not_implemented() { throw std::runtime_error("Not implemented"); }
+
+struct NG_MPI_Status {
+  uintptr_t data[4];
+};
+
+struct NG_MPI_Comm {
+  uintptr_t value;
+  NG_MPI_Comm() { value = 0; }
+  NG_MPI_Comm(uintptr_t value_) : value(value_) {}
+  NG_MPI_Comm(const NG_MPI_Comm &comm) : value(comm.value) {}
+
+  void operator=(int value_) { value = value_; }
+  void operator=(uintptr_t value_) { value = value_; }
+  bool operator==(const NG_MPI_Comm &comm) const { return value == comm.value; }
+  bool operator!=(const NG_MPI_Comm &comm) const { return value != comm.value; }
+};
+
+struct NG_MPI_Datatype {
+  uintptr_t value = 0;
+  NG_MPI_Datatype() = default;
+  NG_MPI_Datatype(uintptr_t value_) : value(value_) {}
+  operator bool() const { return value != 0; }
+  void operator=(NG_MPI_Datatype type) { value = type.value; }
+  void operator=(uintptr_t value_) { value = value_; }
+  void operator=(void *value_) { value = reinterpret_cast<uintptr_t>(value_); }
+};
+
+struct NG_MPI_Request {
+  uintptr_t value = 0;
+  NG_MPI_Request() = default;
+  NG_MPI_Request(uintptr_t value_) : value(value_) {}
+  void operator=(uintptr_t value_) { value = value_; }
+  void operator=(void *value_) { value = reinterpret_cast<uintptr_t>(value_); }
+};
+
+struct NG_MPI_Op {
+  uintptr_t value;
+  NG_MPI_Op(uintptr_t value_) : value(value_) {}
+  void operator=(uintptr_t value_) { value = value_; }
+  void operator=(void *value_) { value = reinterpret_cast<uintptr_t>(value_); }
+};
+
+struct NG_MPI_Group {
+  uintptr_t value = 0;
+  NG_MPI_Group(uintptr_t value_) : value(value_) {}
+  NG_MPI_Group() = default;
+};
+
+struct NG_MPI_Aint {
+  intptr_t value = 0;
+  NG_MPI_Aint(intptr_t value_) : value(value_) {}
+  NG_MPI_Aint() = default;
+};
+
+#else   // NG_MPI_WRAPPER
+using NG_MPI_Comm = MPI_Comm;
+using NG_MPI_Status = MPI_Status;
+using NG_MPI_Datatype = MPI_Datatype;
+using NG_MPI_Request = MPI_Request;
+using NG_MPI_Op = MPI_Op;
+using NG_MPI_Group = MPI_Group;
+using NG_MPI_Aint = MPI_Aint;
+#endif  // NG_MPI_WRAPPER
+
+#include "ng_mpi_generated_declarations.hpp"
+
+}  // namespace ngcore
+
+#endif  // PARALLEL
+#endif  // NG_MPI_HPP_INCLUDED

libsrc/core/ng_mpi_generated_declarations.hpp

@@ -0,0 +1,155 @@
+#ifdef NG_MPI_WRAPPER
+NGCORE_API extern double (*NG_MPI_Wtime)();
+NGCORE_API extern int (*NG_MPI_Allgather)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Allreduce)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Alltoall)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Barrier)(NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Bcast)(void*, int, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Ibcast)(void*, int, NG_MPI_Datatype, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Comm_c2f)(NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Comm_create)(NG_MPI_Comm, NG_MPI_Group, NG_MPI_Comm*);
+NGCORE_API extern int (*NG_MPI_Comm_create_group)(NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*);
+NGCORE_API extern int (*NG_MPI_Comm_free)(NG_MPI_Comm*);
+NGCORE_API extern int (*NG_MPI_Comm_group)(NG_MPI_Comm, NG_MPI_Group*);
+NGCORE_API extern int (*NG_MPI_Comm_rank)(NG_MPI_Comm, int*);
+NGCORE_API extern int (*NG_MPI_Comm_size)(NG_MPI_Comm, int*);
+NGCORE_API extern int (*NG_MPI_Finalize)();
+NGCORE_API extern int (*NG_MPI_Gather)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Gatherv)(void*, int, NG_MPI_Datatype, void*, int*, int*, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Get_count)(NG_MPI_Status*, NG_MPI_Datatype, int*);
+NGCORE_API extern int (*NG_MPI_Get_processor_name)(char*, int*);
+NGCORE_API extern int (*NG_MPI_Group_incl)(NG_MPI_Group, int, int*, NG_MPI_Group*);
+NGCORE_API extern int (*NG_MPI_Init)(int*, char***);
+NGCORE_API extern int (*NG_MPI_Init_thread)(int*, char***, int, int*);
+NGCORE_API extern int (*NG_MPI_Initialized)(int*);
+NGCORE_API extern int (*NG_MPI_Iprobe)(int, int, NG_MPI_Comm, int*, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Irecv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Isend)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Probe)(int, int, NG_MPI_Comm, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Query_thread)(int*);
+NGCORE_API extern int (*NG_MPI_Recv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Recv_init)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Reduce)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Reduce_local)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op);
+NGCORE_API extern int (*NG_MPI_Request_free)(NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Scatter)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Send)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Send_init)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Startall)(int, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Type_commit)(NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_contiguous)(int, NG_MPI_Datatype, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_create_resized)(NG_MPI_Datatype, NG_MPI_Aint, NG_MPI_Aint, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_create_struct)(int, int*, NG_MPI_Aint*, NG_MPI_Datatype*, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_free)(NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_get_extent)(NG_MPI_Datatype, NG_MPI_Aint*, NG_MPI_Aint*);
+NGCORE_API extern int (*NG_MPI_Type_indexed)(int, int*, int*, NG_MPI_Datatype, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_size)(NG_MPI_Datatype, int*);
+NGCORE_API extern int (*NG_MPI_Wait)(NG_MPI_Request*, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Waitall)(int, NG_MPI_Request*, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Waitany)(int, NG_MPI_Request*, int*, NG_MPI_Status*);
+NGCORE_API extern NG_MPI_Comm NG_MPI_COMM_NULL;
+NGCORE_API extern NG_MPI_Comm NG_MPI_COMM_WORLD;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_CHAR;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_C_BOOL;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_DATATYPE_NULL;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_DOUBLE;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_FLOAT;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_INT;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_SHORT;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_UINT64_T;
+NGCORE_API extern NG_MPI_Op NG_MPI_LOR;
+NGCORE_API extern NG_MPI_Op NG_MPI_MAX;
+NGCORE_API extern NG_MPI_Op NG_MPI_MIN;
+NGCORE_API extern NG_MPI_Op NG_MPI_SUM;
+NGCORE_API extern NG_MPI_Request NG_MPI_REQUEST_NULL;
+NGCORE_API extern NG_MPI_Status* NG_MPI_STATUSES_IGNORE;
+NGCORE_API extern NG_MPI_Status* NG_MPI_STATUS_IGNORE;
+NGCORE_API extern int NG_MPI_ANY_SOURCE;
+NGCORE_API extern int NG_MPI_ANY_TAG;
+NGCORE_API extern int NG_MPI_MAX_PROCESSOR_NAME;
+NGCORE_API extern int NG_MPI_PROC_NULL;
+NGCORE_API extern int NG_MPI_ROOT;
+NGCORE_API extern int NG_MPI_SUBVERSION;
+NGCORE_API extern int NG_MPI_THREAD_MULTIPLE;
+NGCORE_API extern int NG_MPI_THREAD_SINGLE;
+NGCORE_API extern int NG_MPI_VERSION;
+NGCORE_API extern void* NG_MPI_IN_PLACE;
+#else  // NG_MPI_WRAPPER
+#define NG_MPI_Wtime MPI_Wtime
+#define NG_MPI_Allgather MPI_Allgather
+#define NG_MPI_Allreduce MPI_Allreduce
+#define NG_MPI_Alltoall MPI_Alltoall
+#define NG_MPI_Barrier MPI_Barrier
+#define NG_MPI_Bcast MPI_Bcast
+#define NG_MPI_Ibcast MPI_Ibcast
+#define NG_MPI_Comm_c2f MPI_Comm_c2f
+#define NG_MPI_Comm_create MPI_Comm_create
+#define NG_MPI_Comm_create_group MPI_Comm_create_group
+#define NG_MPI_Comm_free MPI_Comm_free
+#define NG_MPI_Comm_group MPI_Comm_group
+#define NG_MPI_Comm_rank MPI_Comm_rank
+#define NG_MPI_Comm_size MPI_Comm_size
+#define NG_MPI_Finalize MPI_Finalize
+#define NG_MPI_Gather MPI_Gather
+#define NG_MPI_Gatherv MPI_Gatherv
+#define NG_MPI_Get_count MPI_Get_count
+#define NG_MPI_Get_processor_name MPI_Get_processor_name
+#define NG_MPI_Group_incl MPI_Group_incl
+#define NG_MPI_Init MPI_Init
+#define NG_MPI_Init_thread MPI_Init_thread
+#define NG_MPI_Initialized MPI_Initialized
+#define NG_MPI_Iprobe MPI_Iprobe
+#define NG_MPI_Irecv MPI_Irecv
+#define NG_MPI_Isend MPI_Isend
+#define NG_MPI_Probe MPI_Probe
+#define NG_MPI_Query_thread MPI_Query_thread
+#define NG_MPI_Recv MPI_Recv
+#define NG_MPI_Recv_init MPI_Recv_init
+#define NG_MPI_Reduce MPI_Reduce
+#define NG_MPI_Reduce_local MPI_Reduce_local
+#define NG_MPI_Request_free MPI_Request_free
+#define NG_MPI_Scatter MPI_Scatter
+#define NG_MPI_Send MPI_Send
+#define NG_MPI_Send_init MPI_Send_init
+#define NG_MPI_Startall MPI_Startall
+#define NG_MPI_Type_commit MPI_Type_commit
+#define NG_MPI_Type_contiguous MPI_Type_contiguous
+#define NG_MPI_Type_create_resized MPI_Type_create_resized
+#define NG_MPI_Type_create_struct MPI_Type_create_struct
+#define NG_MPI_Type_free MPI_Type_free
+#define NG_MPI_Type_get_extent MPI_Type_get_extent
+#define NG_MPI_Type_indexed MPI_Type_indexed
+#define NG_MPI_Type_size MPI_Type_size
+#define NG_MPI_Wait MPI_Wait
+#define NG_MPI_Waitall MPI_Waitall
+#define NG_MPI_Waitany MPI_Waitany
+#define NG_MPI_COMM_NULL MPI_COMM_NULL
+#define NG_MPI_COMM_WORLD MPI_COMM_WORLD
+#define NG_MPI_CHAR MPI_CHAR
+#define NG_MPI_CXX_DOUBLE_COMPLEX MPI_CXX_DOUBLE_COMPLEX
+#define NG_MPI_C_BOOL MPI_C_BOOL
+#define NG_MPI_DATATYPE_NULL MPI_DATATYPE_NULL
+#define NG_MPI_DOUBLE MPI_DOUBLE
+#define NG_MPI_FLOAT MPI_FLOAT
+#define NG_MPI_INT MPI_INT
+#define NG_MPI_SHORT MPI_SHORT
+#define NG_MPI_UINT64_T MPI_UINT64_T
+#define NG_MPI_LOR MPI_LOR
+#define NG_MPI_MAX MPI_MAX
+#define NG_MPI_MIN MPI_MIN
+#define NG_MPI_SUM MPI_SUM
+#define NG_MPI_REQUEST_NULL MPI_REQUEST_NULL
+#define NG_MPI_STATUSES_IGNORE MPI_STATUSES_IGNORE
+#define NG_MPI_STATUS_IGNORE MPI_STATUS_IGNORE
+#define NG_MPI_ANY_SOURCE MPI_ANY_SOURCE
+#define NG_MPI_ANY_TAG MPI_ANY_TAG
+#define NG_MPI_MAX_PROCESSOR_NAME MPI_MAX_PROCESSOR_NAME
+#define NG_MPI_PROC_NULL MPI_PROC_NULL
+#define NG_MPI_ROOT MPI_ROOT
+#define NG_MPI_SUBVERSION MPI_SUBVERSION
+#define NG_MPI_THREAD_MULTIPLE MPI_THREAD_MULTIPLE
+#define NG_MPI_THREAD_SINGLE MPI_THREAD_SINGLE
+#define NG_MPI_VERSION MPI_VERSION
+#define NG_MPI_IN_PLACE MPI_IN_PLACE
+#endif // NG_MPI_WRAPPER

libsrc/core/ng_mpi_generated_dummy_init.hpp

@@ -0,0 +1,76 @@
+decltype(NG_MPI_Wtime) NG_MPI_Wtime = []()->double { throw no_mpi(); };
+decltype(NG_MPI_Allgather) NG_MPI_Allgather = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Allreduce) NG_MPI_Allreduce = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Alltoall) NG_MPI_Alltoall = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Barrier) NG_MPI_Barrier = [](NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Bcast) NG_MPI_Bcast = [](void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Ibcast) NG_MPI_Ibcast = [](void*, int, NG_MPI_Datatype, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_c2f) NG_MPI_Comm_c2f = [](NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_create) NG_MPI_Comm_create = [](NG_MPI_Comm, NG_MPI_Group, NG_MPI_Comm*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_create_group) NG_MPI_Comm_create_group = [](NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_free) NG_MPI_Comm_free = [](NG_MPI_Comm*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_group) NG_MPI_Comm_group = [](NG_MPI_Comm, NG_MPI_Group*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_rank) NG_MPI_Comm_rank = [](NG_MPI_Comm, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_size) NG_MPI_Comm_size = [](NG_MPI_Comm, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Finalize) NG_MPI_Finalize = []()->int { throw no_mpi(); };
+decltype(NG_MPI_Gather) NG_MPI_Gather = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Gatherv) NG_MPI_Gatherv = [](void*, int, NG_MPI_Datatype, void*, int*, int*, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Get_count) NG_MPI_Get_count = [](NG_MPI_Status*, NG_MPI_Datatype, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Get_processor_name) NG_MPI_Get_processor_name = [](char*, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Group_incl) NG_MPI_Group_incl = [](NG_MPI_Group, int, int*, NG_MPI_Group*)->int { throw no_mpi(); };
+decltype(NG_MPI_Init) NG_MPI_Init = [](int*, char***)->int { throw no_mpi(); };
+decltype(NG_MPI_Init_thread) NG_MPI_Init_thread = [](int*, char***, int, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Initialized) NG_MPI_Initialized = [](int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Iprobe) NG_MPI_Iprobe = [](int, int, NG_MPI_Comm, int*, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Irecv) NG_MPI_Irecv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Isend) NG_MPI_Isend = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Probe) NG_MPI_Probe = [](int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Query_thread) NG_MPI_Query_thread = [](int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Recv) NG_MPI_Recv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Recv_init) NG_MPI_Recv_init = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Reduce) NG_MPI_Reduce = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Reduce_local) NG_MPI_Reduce_local = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op)->int { throw no_mpi(); };
+decltype(NG_MPI_Request_free) NG_MPI_Request_free = [](NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Scatter) NG_MPI_Scatter = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Send) NG_MPI_Send = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Send_init) NG_MPI_Send_init = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Startall) NG_MPI_Startall = [](int, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_commit) NG_MPI_Type_commit = [](NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_contiguous) NG_MPI_Type_contiguous = [](int, NG_MPI_Datatype, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_create_resized) NG_MPI_Type_create_resized = [](NG_MPI_Datatype, NG_MPI_Aint, NG_MPI_Aint, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_create_struct) NG_MPI_Type_create_struct = [](int, int*, NG_MPI_Aint*, NG_MPI_Datatype*, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_free) NG_MPI_Type_free = [](NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_get_extent) NG_MPI_Type_get_extent = [](NG_MPI_Datatype, NG_MPI_Aint*, NG_MPI_Aint*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_indexed) NG_MPI_Type_indexed = [](int, int*, int*, NG_MPI_Datatype, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_size) NG_MPI_Type_size = [](NG_MPI_Datatype, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Wait) NG_MPI_Wait = [](NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Waitall) NG_MPI_Waitall = [](int, NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Waitany) NG_MPI_Waitany = [](int, NG_MPI_Request*, int*, NG_MPI_Status*)->int { throw no_mpi(); };
+NG_MPI_Comm NG_MPI_COMM_NULL = 0;
+NG_MPI_Comm NG_MPI_COMM_WORLD = 0;
+NG_MPI_Datatype NG_MPI_CHAR = 0;
+NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX = 0;
+NG_MPI_Datatype NG_MPI_C_BOOL = 0;
+NG_MPI_Datatype NG_MPI_DATATYPE_NULL = 0;
+NG_MPI_Datatype NG_MPI_DOUBLE = 0;
+NG_MPI_Datatype NG_MPI_FLOAT = 0;
+NG_MPI_Datatype NG_MPI_INT = 0;
+NG_MPI_Datatype NG_MPI_SHORT = 0;
+NG_MPI_Datatype NG_MPI_UINT64_T = 0;
+NG_MPI_Op NG_MPI_LOR = 0;
+NG_MPI_Op NG_MPI_MAX = 0;
+NG_MPI_Op NG_MPI_MIN = 0;
+NG_MPI_Op NG_MPI_SUM = 0;
+NG_MPI_Request NG_MPI_REQUEST_NULL = 0;
+NG_MPI_Status* NG_MPI_STATUSES_IGNORE = 0;
+NG_MPI_Status* NG_MPI_STATUS_IGNORE = 0;
+int NG_MPI_ANY_SOURCE = 0;
+int NG_MPI_ANY_TAG = 0;
+int NG_MPI_MAX_PROCESSOR_NAME = 0;
+int NG_MPI_PROC_NULL = 0;
+int NG_MPI_ROOT = 0;
+int NG_MPI_SUBVERSION = 0;
+int NG_MPI_THREAD_MULTIPLE = 0;
+int NG_MPI_THREAD_SINGLE = 0;
+int NG_MPI_VERSION = 0;
+void* NG_MPI_IN_PLACE = 0;

libsrc/core/ng_mpi_generated_init.hpp

@@ -0,0 +1,76 @@
+NG_MPI_Wtime = []()->double { return MPI_Wtime(); };
+NG_MPI_Allgather = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, NG_MPI_Comm arg6)->int { return MPI_Allgather( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Allreduce = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4, NG_MPI_Comm arg5)->int { return MPI_Allreduce( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4),  ng2mpi(arg5)); };
+NG_MPI_Alltoall = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, NG_MPI_Comm arg6)->int { return MPI_Alltoall( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Barrier = [](NG_MPI_Comm arg0)->int { return MPI_Barrier( ng2mpi(arg0)); };
+NG_MPI_Bcast = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, NG_MPI_Comm arg4)->int { return MPI_Bcast( arg0,  arg1,  ng2mpi(arg2),  arg3,  ng2mpi(arg4)); };
+NG_MPI_Ibcast = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, NG_MPI_Comm arg4, NG_MPI_Request* arg5)->int { return MPI_Ibcast( arg0,  arg1,  ng2mpi(arg2),  arg3,  ng2mpi(arg4),  ng2mpi(arg5)); };
+NG_MPI_Comm_c2f = [](NG_MPI_Comm arg0)->int { return MPI_Comm_c2f( ng2mpi(arg0)); };
+NG_MPI_Comm_create = [](NG_MPI_Comm arg0, NG_MPI_Group arg1, NG_MPI_Comm* arg2)->int { return MPI_Comm_create( ng2mpi(arg0),  ng2mpi(arg1),  ng2mpi(arg2)); };
+NG_MPI_Comm_create_group = [](NG_MPI_Comm arg0, NG_MPI_Group arg1, int arg2, NG_MPI_Comm* arg3)->int { return MPI_Comm_create_group( ng2mpi(arg0),  ng2mpi(arg1),  arg2,  ng2mpi(arg3)); };
+NG_MPI_Comm_free = [](NG_MPI_Comm* arg0)->int { return MPI_Comm_free( ng2mpi(arg0)); };
+NG_MPI_Comm_group = [](NG_MPI_Comm arg0, NG_MPI_Group* arg1)->int { return MPI_Comm_group( ng2mpi(arg0),  ng2mpi(arg1)); };
+NG_MPI_Comm_rank = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_rank( ng2mpi(arg0),  arg1); };
+NG_MPI_Comm_size = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_size( ng2mpi(arg0),  arg1); };
+NG_MPI_Finalize = []()->int { return MPI_Finalize(); };
+NG_MPI_Gather = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, int arg6, NG_MPI_Comm arg7)->int { return MPI_Gather( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  arg6,  ng2mpi(arg7)); };
+NG_MPI_Gatherv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int* arg4, int* arg5, NG_MPI_Datatype arg6, int arg7, NG_MPI_Comm arg8)->int { return MPI_Gatherv( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  arg5,  ng2mpi(arg6),  arg7,  ng2mpi(arg8)); };
+NG_MPI_Get_count = [](NG_MPI_Status* arg0, NG_MPI_Datatype arg1, int* arg2)->int { return MPI_Get_count( ng2mpi(arg0),  ng2mpi(arg1),  arg2); };
+NG_MPI_Get_processor_name = [](char* arg0, int* arg1)->int { return MPI_Get_processor_name( arg0,  arg1); };
+NG_MPI_Group_incl = [](NG_MPI_Group arg0, int arg1, int* arg2, NG_MPI_Group* arg3)->int { return MPI_Group_incl( ng2mpi(arg0),  arg1,  arg2,  ng2mpi(arg3)); };
+NG_MPI_Init = [](int* arg0, char*** arg1)->int { return MPI_Init( arg0,  arg1); };
+NG_MPI_Init_thread = [](int* arg0, char*** arg1, int arg2, int* arg3)->int { return MPI_Init_thread( arg0,  arg1,  arg2,  arg3); };
+NG_MPI_Initialized = [](int* arg0)->int { return MPI_Initialized( arg0); };
+NG_MPI_Iprobe = [](int arg0, int arg1, NG_MPI_Comm arg2, int* arg3, NG_MPI_Status* arg4)->int { return MPI_Iprobe( arg0,  arg1,  ng2mpi(arg2),  arg3,  ng2mpi(arg4)); };
+NG_MPI_Irecv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Irecv( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Isend = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Isend( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Probe = [](int arg0, int arg1, NG_MPI_Comm arg2, NG_MPI_Status* arg3)->int { return MPI_Probe( arg0,  arg1,  ng2mpi(arg2),  ng2mpi(arg3)); };
+NG_MPI_Query_thread = [](int* arg0)->int { return MPI_Query_thread( arg0); };
+NG_MPI_Recv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Status* arg6)->int { return MPI_Recv( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Recv_init = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Recv_init( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Reduce = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4, int arg5, NG_MPI_Comm arg6)->int { return MPI_Reduce( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4),  arg5,  ng2mpi(arg6)); };
+NG_MPI_Reduce_local = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4)->int { return MPI_Reduce_local( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4)); };
+NG_MPI_Request_free = [](NG_MPI_Request* arg0)->int { return MPI_Request_free( ng2mpi(arg0)); };
+NG_MPI_Scatter = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, int arg6, NG_MPI_Comm arg7)->int { return MPI_Scatter( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  arg6,  ng2mpi(arg7)); };
+NG_MPI_Send = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5)->int { return MPI_Send( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5)); };
+NG_MPI_Send_init = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Send_init( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Startall = [](int arg0, NG_MPI_Request* arg1)->int { return MPI_Startall( arg0,  ng2mpi(arg1, arg0)); };
+NG_MPI_Type_commit = [](NG_MPI_Datatype* arg0)->int { return MPI_Type_commit( ng2mpi(arg0)); };
+NG_MPI_Type_contiguous = [](int arg0, NG_MPI_Datatype arg1, NG_MPI_Datatype* arg2)->int { return MPI_Type_contiguous( arg0,  ng2mpi(arg1),  ng2mpi(arg2)); };
+NG_MPI_Type_create_resized = [](NG_MPI_Datatype arg0, NG_MPI_Aint arg1, NG_MPI_Aint arg2, NG_MPI_Datatype* arg3)->int { return MPI_Type_create_resized( ng2mpi(arg0),  ng2mpi(arg1),  ng2mpi(arg2),  ng2mpi(arg3)); };
+NG_MPI_Type_create_struct = [](int arg0, int* arg1, NG_MPI_Aint* arg2, NG_MPI_Datatype* arg3, NG_MPI_Datatype* arg4)->int { return MPI_Type_create_struct( arg0,  arg1,  ng2mpi(arg2, arg0),  ng2mpi(arg3, arg0),  ng2mpi(arg4)); };
+NG_MPI_Type_free = [](NG_MPI_Datatype* arg0)->int { return MPI_Type_free( ng2mpi(arg0)); };
+NG_MPI_Type_get_extent = [](NG_MPI_Datatype arg0, NG_MPI_Aint* arg1, NG_MPI_Aint* arg2)->int { return MPI_Type_get_extent( ng2mpi(arg0),  ng2mpi(arg1),  ng2mpi(arg2)); };
+NG_MPI_Type_indexed = [](int arg0, int* arg1, int* arg2, NG_MPI_Datatype arg3, NG_MPI_Datatype* arg4)->int { return MPI_Type_indexed( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4)); };
+NG_MPI_Type_size = [](NG_MPI_Datatype arg0, int* arg1)->int { return MPI_Type_size( ng2mpi(arg0),  arg1); };
+NG_MPI_Wait = [](NG_MPI_Request* arg0, NG_MPI_Status* arg1)->int { return MPI_Wait( ng2mpi(arg0),  ng2mpi(arg1)); };
+NG_MPI_Waitall = [](int arg0, NG_MPI_Request* arg1, NG_MPI_Status* arg2)->int { return MPI_Waitall( arg0,  ng2mpi(arg1, arg0),  ng2mpi(arg2)); };
+NG_MPI_Waitany = [](int arg0, NG_MPI_Request* arg1, int* arg2, NG_MPI_Status* arg3)->int { return MPI_Waitany( arg0,  ng2mpi(arg1, arg0),  arg2,  ng2mpi(arg3)); };
+NG_MPI_COMM_NULL = mpi2ng(MPI_COMM_NULL);
+NG_MPI_COMM_WORLD = mpi2ng(MPI_COMM_WORLD);
+NG_MPI_CHAR = mpi2ng(MPI_CHAR);
+NG_MPI_CXX_DOUBLE_COMPLEX = mpi2ng(MPI_CXX_DOUBLE_COMPLEX);
+NG_MPI_C_BOOL = mpi2ng(MPI_C_BOOL);
+NG_MPI_DATATYPE_NULL = mpi2ng(MPI_DATATYPE_NULL);
+NG_MPI_DOUBLE = mpi2ng(MPI_DOUBLE);
+NG_MPI_FLOAT = mpi2ng(MPI_FLOAT);
+NG_MPI_INT = mpi2ng(MPI_INT);
+NG_MPI_SHORT = mpi2ng(MPI_SHORT);
+NG_MPI_UINT64_T = mpi2ng(MPI_UINT64_T);
+NG_MPI_LOR = mpi2ng(MPI_LOR);
+NG_MPI_MAX = mpi2ng(MPI_MAX);
+NG_MPI_MIN = mpi2ng(MPI_MIN);
+NG_MPI_SUM = mpi2ng(MPI_SUM);
+NG_MPI_REQUEST_NULL = mpi2ng(MPI_REQUEST_NULL);
+NG_MPI_STATUSES_IGNORE = mpi2ng(MPI_STATUSES_IGNORE);
+NG_MPI_STATUS_IGNORE = mpi2ng(MPI_STATUS_IGNORE);
+NG_MPI_ANY_SOURCE = mpi2ng(MPI_ANY_SOURCE);
+NG_MPI_ANY_TAG = mpi2ng(MPI_ANY_TAG);
+NG_MPI_MAX_PROCESSOR_NAME = mpi2ng(MPI_MAX_PROCESSOR_NAME);
+NG_MPI_PROC_NULL = mpi2ng(MPI_PROC_NULL);
+NG_MPI_ROOT = mpi2ng(MPI_ROOT);
+NG_MPI_SUBVERSION = mpi2ng(MPI_SUBVERSION);
+NG_MPI_THREAD_MULTIPLE = mpi2ng(MPI_THREAD_MULTIPLE);
+NG_MPI_THREAD_SINGLE = mpi2ng(MPI_THREAD_SINGLE);
+NG_MPI_VERSION = mpi2ng(MPI_VERSION);
+NG_MPI_IN_PLACE = mpi2ng(MPI_IN_PLACE);

libsrc/core/ng_mpi_native.hpp

@@ -0,0 +1,21 @@
+#ifndef NG_MPI_NATIVE_HPP
+#define NG_MPI_NATIVE_HPP
+
+#include <mpi.h>
+
+#include "mpi_wrapper.hpp"
+#include "ng_mpi.hpp"
+
+namespace ngcore {
+
+MPI_Comm NG_MPI_Native(NG_MPI_Comm comm) {
+  return reinterpret_cast<MPI_Comm>(comm.value);
+}
+
+MPI_Comm NG_MPI_Native(NgMPI_Comm comm) {
+  return reinterpret_cast<MPI_Comm>(static_cast<NG_MPI_Comm>(comm).value);
+}
+
+}  // namespace ngcore
+
+#endif  // NG_MPI_NATIVE_HPP

libsrc/core/ng_mpi_wrapper.cpp

@@ -0,0 +1,206 @@
+#ifdef PARALLEL
+
+#include <filesystem>
+#include <iostream>
+#include <stdexcept>
+
+#include "ng_mpi.hpp"
+#include "ngstream.hpp"
+#ifdef NG_PYTHON
+#include "python_ngcore.hpp"
+#endif // NG_PYTHON
+#include "utils.hpp"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+
+#ifndef NG_MPI_WRAPPER
+#ifdef NG_PYTHON
+#define MPI4PY_LIMITED_API 1
+#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1
+#define MPI4PY_LIMITED_API_SKIP_SESSION 1
+#include "mpi4py_pycapi.h"  // mpi4py < 4.0.0
+#endif // NG_PYTHON
+#endif // NG_MPI_WRAPPER
+
+namespace ngcore {
+
+#ifdef NG_MPI_WRAPPER
+static std::unique_ptr<SharedLibrary> mpi_lib, ng_mpi_lib;
+static bool need_mpi_finalize = false;
+
+struct MPIFinalizer {
+  ~MPIFinalizer() {
+    if (need_mpi_finalize) {
+      cout << IM(5) << "Calling MPI_Finalize" << endl;
+      NG_MPI_Finalize();
+    }
+  }
+} mpi_finalizer;
+
+bool MPI_Loaded() { return ng_mpi_lib != nullptr; }
+
+void InitMPI(std::optional<std::filesystem::path> mpi_lib_path) {
+  if (ng_mpi_lib) return;
+
+  cout << IM(3) << "InitMPI" << endl;
+
+  std::string vendor = "";
+  std::string mpi4py_lib_file = "";
+
+  if (mpi_lib_path) {
+    // Dynamic load of given shared MPI library
+    // Then call MPI_Init, read the library version and set the vender name
+    try {
+      typedef int (*init_handle)(int *, char ***);
+      typedef int (*mpi_initialized_handle)(int *);
+      mpi_lib =
+          std::make_unique<SharedLibrary>(*mpi_lib_path, std::nullopt, true);
+      auto mpi_init = mpi_lib->GetSymbol<init_handle>("MPI_Init");
+      auto mpi_initialized =
+          mpi_lib->GetSymbol<mpi_initialized_handle>("MPI_Initialized");
+
+      int flag = 0;
+      mpi_initialized(&flag);
+      if (!flag) {
+        typedef const char *pchar;
+        int argc = 1;
+        pchar args[] = {"netgen", nullptr};
+        pchar *argv = &args[0];
+        cout << IM(5) << "Calling MPI_Init" << endl;
+        mpi_init(&argc, (char ***)argv);
+        need_mpi_finalize = true;
+      }
+
+      char c_version_string[65536];
+      c_version_string[0] = '\0';
+      int result_len = 0;
+      typedef void (*get_version_handle)(char *, int *);
+      auto get_version =
+          mpi_lib->GetSymbol<get_version_handle>("MPI_Get_library_version");
+      get_version(c_version_string, &result_len);
+      std::string version = c_version_string;
+
+      if (version.substr(0, 8) == "Open MPI")
+        vendor = "Open MPI";
+      else if (version.substr(0, 5) == "MPICH")
+        vendor = "MPICH";
+      else if (version.substr(0, 13) == "Microsoft MPI")
+        vendor = "Microsoft MPI";
+      else if (version.substr(0, 12) == "Intel(R) MPI")
+        vendor = "Intel MPI";
+      else
+        throw std::runtime_error(
+            std::string("Unknown MPI version: " + version));
+    } catch (std::runtime_error &e) {
+      cerr << "Could not load MPI: " << e.what() << endl;
+      throw e;
+    }
+  } else {
+#ifdef NG_PYTHON
+    // Use mpi4py to init MPI library and get the vendor name
+    auto mpi4py = py::module::import("mpi4py.MPI");
+    vendor = mpi4py.attr("get_vendor")()[py::int_(0)].cast<std::string>();
+
+#ifndef WIN32
+    // Load mpi4py library (it exports all MPI symbols) to have all MPI symbols
+    // available before the ng_mpi wrapper is loaded This is not necessary on
+    // windows as the matching mpi dll is linked to the ng_mpi wrapper directly
+    mpi4py_lib_file = mpi4py.attr("__file__").cast<std::string>();
+    mpi_lib =
+        std::make_unique<SharedLibrary>(mpi4py_lib_file, std::nullopt, true);
+#endif  // WIN32
+#endif // NG_PYTHON
+  }
+
+  std::string ng_lib_name = "";
+  if (vendor == "Open MPI")
+    ng_lib_name = "ng_openmpi";
+  else if (vendor == "MPICH")
+    ng_lib_name = "ng_mpich";
+  else if (vendor == "Microsoft MPI")
+    ng_lib_name = "ng_microsoft_mpi";
+  else if (vendor == "Intel MPI")
+    ng_lib_name = "ng_intel_mpi";
+  else
+    throw std::runtime_error("Unknown MPI vendor: " + vendor);
+
+  ng_lib_name += NETGEN_SHARED_LIBRARY_SUFFIX;
+
+  // Load the ng_mpi wrapper and call ng_init_mpi to set all function pointers
+  typedef void (*ng_init_handle)();
+  ng_mpi_lib = std::make_unique<SharedLibrary>(ng_lib_name);
+  ng_mpi_lib->GetSymbol<ng_init_handle>("ng_init_mpi")();
+  std::cout << IM(3) << "MPI wrapper loaded, vendor: " << vendor << endl;
+}
+
+static std::runtime_error no_mpi() {
+  return std::runtime_error("MPI not enabled");
+}
+
+#ifdef NG_PYTHON
+decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
+    [](py::handle py_obj, NG_MPI_Comm &ng_comm) -> bool {
+  // If this gets called, it means that we want to convert an mpi4py
+  // communicator to a Netgen MPI communicator, but the Netgen MPI wrapper
+  // runtime was not yet initialized.
+
+  // store the current address of this function
+  auto old_converter_address = NG_MPI_CommFromMPI4Py;
+
+  // initialize the MPI wrapper runtime, this sets all the function pointers
+  InitMPI();
+
+  // if the initialization was successful, the function pointer should have
+  // changed
+  // -> call the actual conversion function
+  if (NG_MPI_CommFromMPI4Py != old_converter_address)
+    return NG_MPI_CommFromMPI4Py(py_obj, ng_comm);
+
+  // otherwise, something strange happened
+  throw no_mpi();
+};
+decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
+    [](NG_MPI_Comm) -> py::handle { throw no_mpi(); };
+#endif  // NG_PYTHON
+
+#include "ng_mpi_generated_dummy_init.hpp"
+#else  // NG_MPI_WRAPPER
+
+static bool imported_mpi4py = false;
+#ifdef NG_PYTHON
+decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
+    [](py::handle src, NG_MPI_Comm &dst) -> bool {
+  if (!imported_mpi4py) {
+    import_mpi4py__MPI();
+    imported_mpi4py = true;
+  }
+  PyObject *py_src = src.ptr();
+  auto type = Py_TYPE(py_src);
+  if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
+    dst = *PyMPIComm_Get(py_src);
+    return !PyErr_Occurred();
+  }
+  return false;
+};
+
+decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
+    [](NG_MPI_Comm src) -> py::handle {
+  if (!imported_mpi4py) {
+    import_mpi4py__MPI();
+    imported_mpi4py = true;
+  }
+  return py::handle(PyMPIComm_New(src));
+};
+
+#endif  // NG_PYTHON
+
+bool MPI_Loaded() { return true; }
+void InitMPI(std::optional<std::filesystem::path>) {}
+
+#endif  // NG_MPI_WRAPPER
+
+}  // namespace ngcore
+
+#endif  // PARALLEL

libsrc/core/ngcore.hpp

@@ -10,7 +10,7 @@
 #include "hashtable.hpp"
 #include "localheap.hpp"
 #include "logging.hpp"
-#include "mpi_wrapper.hpp"
+// #include "mpi_wrapper.hpp"
 #include "profiler.hpp"
 #include "signal.hpp"
 #include "simd.hpp"
@@ -22,5 +22,6 @@
 #include "xbool.hpp"
 #include "ngstream.hpp"
 #include "utils.hpp"
+#include "ranges.hpp"
 
 #endif // NETGEN_CORE_NGCORE_HPP

libsrc/core/ngcore_api.hpp

@@ -1,6 +1,8 @@
 #ifndef NETGEN_CORE_NGCORE_API_HPP
 #define NETGEN_CORE_NGCORE_API_HPP
 
+#include "netgen_config.hpp"
+
 #ifdef WIN32
 
 // This function or variable may be unsafe. Consider using _ftime64_s instead. To disable deprecation, use _CRT_SECURE_NO_WARNINGS. See online help for details.

libsrc/core/paje_trace.cpp

@@ -7,12 +7,14 @@
 
 #include "archive.hpp"           // for Demangle
 #include "paje_trace.hpp"
+#include "ng_mpi.hpp"
 #include "profiler.hpp"
 #include "mpi_wrapper.hpp"
 
 extern const char *header;
 
 constexpr int MPI_PAJE_WRITER = 1;
+constexpr int ASSUMED_MPI_MAX_PROCESSOR_NAME = 256;
 
 namespace ngcore
 {
@@ -24,7 +26,7 @@ namespace ngcore
     if(id<NgProfiler::SIZE)
       return NgProfiler::GetName(id);
 
-    NgMPI_Comm comm(MPI_COMM_WORLD);
+    NgMPI_Comm comm(NG_MPI_COMM_WORLD);
     return NgProfiler::GetName(id-NgProfiler::SIZE*comm.Rank());
 #endif // PARALLEL
   }
@@ -39,6 +41,7 @@ namespace ngcore
   bool PajeTrace::trace_thread_counter = false;
   bool PajeTrace::trace_threads = true;
   bool PajeTrace::mem_tracing_enabled = true;
+  bool PajeTrace::write_paje_file = true;
 
   PajeTrace :: PajeTrace(int anthreads, std::string aname)
   {
@@ -70,9 +73,12 @@ namespace ngcore
 
     // sync start time when running in parallel
 #ifdef PARALLEL
-    NgMPI_Comm comm(MPI_COMM_WORLD);
+    if(MPI_Loaded())
+    {
+      NgMPI_Comm comm(NG_MPI_COMM_WORLD);
       for([[maybe_unused]] auto i : Range(5))
           comm.Barrier();
+    }
 #endif // PARALLEL
 
     start_time = GetTimeCounter();
@@ -112,11 +118,14 @@ namespace ngcore
     for(auto i : IntRange(n_memory_events_at_start, memory_events.size()))
       memory_events[i].time -= start_time;
 
-    NgMPI_Comm comm(MPI_COMM_WORLD);
+    NgMPI_Comm comm;
-
+  #ifdef PARALLEL
+    if(MPI_Loaded())
+      comm = NgMPI_Comm(NG_MPI_COMM_WORLD);
+  #endif
     if(comm.Size()==1)
     {
-      Write(tracefile_name);
+      Write();
     }
     else
     {
@@ -128,7 +137,7 @@ namespace ngcore
         event.timer_id += NgProfiler::SIZE*comm.Rank();
 
       if(comm.Rank() == MPI_PAJE_WRITER)
-        Write(tracefile_name);
+        Write();
       else
         SendData();
     }
@@ -435,7 +444,16 @@ namespace ngcore
 
   NGCORE_API PajeTrace *trace;
 
-  void PajeTrace::Write( const std::string & filename )
+  void PajeTrace::Write( )
+    {
+      if(write_paje_file) WritePajeFile( tracefile_name );
+      WriteTimingChart();
+#ifdef NETGEN_TRACE_MEMORY
+      WriteMemoryChart("");
+#endif // NETGEN_TRACE_MEMORY
+    }
+
+  void PajeTrace::WritePajeFile( const std::string & filename )
     {
       auto n_events = jobs.size() + timer_events.size();
       for(auto & vtasks : tasks)
@@ -486,25 +504,25 @@ namespace ngcore
       std::vector <int> thread_aliases;
       std::vector<int> container_nodes;
 
-#ifdef PARALLEL
+      NgMPI_Comm comm;
-      // Hostnames
+  #ifdef PARALLEL
-      NgMPI_Comm comm(MPI_COMM_WORLD);
+      if(MPI_Loaded())
-      // auto rank = comm.Rank();
+        comm = NgMPI_Comm(NG_MPI_COMM_WORLD);
-      auto nranks = comm.Size();
+      if(comm.Size()>1)
-      if(nranks>1)
       {
-        nthreads = nranks;
+        auto comm = NgMPI_Comm(NG_MPI_COMM_WORLD);
+        nthreads = comm.Size();
         thread_aliases.reserve(nthreads);
 
-        std::array<char, MPI_MAX_PROCESSOR_NAME+1> ahostname;
+        std::array<char, ASSUMED_MPI_MAX_PROCESSOR_NAME+1> ahostname;
         int len;
-        MPI_Get_processor_name(ahostname.data(), &len);
+        NG_MPI_Get_processor_name(ahostname.data(), &len);
         std::string hostname = ahostname.data();
 
         std::map<std::string, int> host_map;
 
         std::string name;
-        for(auto i : IntRange(0, nranks))
+        for(auto i : IntRange(0, comm.Size()))
         {
           if(i!=MPI_PAJE_WRITER)
             comm.Recv(name, i, 0);
@@ -519,7 +537,7 @@ namespace ngcore
         }
       }
       else
-#endif // PARALLEL
+  #endif
       {
         container_nodes.reserve(num_nodes);
         for(int i=0; i<num_nodes; i++)
@@ -595,10 +613,9 @@ namespace ngcore
       for(auto id : timer_ids)
           timer_names[id] = GetTimerName(id);
 
-#ifdef PARALLEL
+      if(comm.Size()>1)
-      if(nranks>1)
       {
-        for(auto src : IntRange(0, nranks))
+        for(auto src : IntRange(0, comm.Size()))
         {
           if(src==MPI_PAJE_WRITER)
             continue;
@@ -617,7 +634,6 @@ namespace ngcore
           }
         }
       }
-#endif // PARALLEL
 
       for(auto id : timer_ids)
           timer_aliases[id] = paje.DefineEntityValue( state_type_timer, timer_names[id], -1 );
@@ -742,7 +758,7 @@ namespace ngcore
         }
 
 #ifdef PARALLEL
-      if(nranks>1)
+      if(comm.Size()>1)
       {
         for(auto & event : timer_events)
         {
@@ -758,7 +774,7 @@ namespace ngcore
         Array<bool> is_start;
         Array<int> thread_id;
 
-        for(auto src : IntRange(0, nranks))
+        for(auto src : IntRange(0, comm.Size()))
         {
           if(src==MPI_PAJE_WRITER)
             continue;
@@ -843,10 +859,6 @@ namespace ngcore
                 }
             }
         }
-      WriteTimingChart();
-#ifdef NETGEN_TRACE_MEMORY
-      WriteMemoryChart("");
-#endif // NETGEN_TRACE_MEMORY
       paje.WriteEvents();
     }
 
@@ -854,15 +866,15 @@ namespace ngcore
     {
 #ifdef PARALLEL
       // Hostname
-      NgMPI_Comm comm(MPI_COMM_WORLD);
+      NgMPI_Comm comm(NG_MPI_COMM_WORLD);
       // auto rank = comm.Rank();
       // auto nranks = comm.Size();
 
       std::string hostname;
         {
-          std::array<char, MPI_MAX_PROCESSOR_NAME+1> ahostname;
+          std::array<char, ASSUMED_MPI_MAX_PROCESSOR_NAME+1> ahostname;
           int len;
-          MPI_Get_processor_name(ahostname.data(), &len);
+          NG_MPI_Get_processor_name(ahostname.data(), &len);
           hostname = ahostname.data();
         }
 
@@ -955,10 +967,18 @@ namespace ngcore
     f.precision(4);
     f << R"CODE_(
 <head>
-  <script src="https://d3js.org/d3.v5.min.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/d3@7"></script>
   <script src="https://unpkg.com/sunburst-chart"></script>
 
-  <style>body { margin: 0 }</style>
+  <style>
+    body { margin: 0 }
+    .tooltip {
+      white-space: pre-line !important;
+      max-width: 800px !important;
+      word-wrap: break-word !important;
+      padding: 10px !important;
+    }
+  </style>
 )CODE_";
     if(!time_or_memory)
       f << "<title>Maximum Memory Consumption</title>\n";

libsrc/core/paje_trace.hpp

@@ -25,6 +25,7 @@ namespace ngcore
       NGCORE_API static bool trace_thread_counter;
       NGCORE_API static bool trace_threads;
       NGCORE_API static bool mem_tracing_enabled;
+      NGCORE_API static bool write_paje_file;
 
       bool tracing_enabled;
       TTimePoint start_time;
@@ -32,6 +33,8 @@ namespace ngcore
       size_t n_memory_events_at_start;
 
     public:
+      NGCORE_API void Write();
+      NGCORE_API void WritePajeFile( const std::string & filename );
       NGCORE_API void WriteTimingChart();
 #ifdef NETGEN_TRACE_MEMORY
       NGCORE_API void WriteMemoryChart( std::string fname );
@@ -61,6 +64,11 @@ namespace ngcore
           max_tracefile_size = max_size;
         }
 
+      static void SetWritePajeFile( bool write )
+        {
+          write_paje_file = write;
+        }
+
       std::string tracefile_name;
 
       struct Job
@@ -262,8 +270,6 @@ namespace ngcore
           links[thread_id].push_back( ThreadLink{thread_id, key, GetTimeCounter(), false} );
         }
 
-      void Write( const std::string & filename );
-
       void SendData(); // MPI parallel data reduction
 
     };

libsrc/core/profiler.cpp

@@ -116,6 +116,7 @@ namespace ngcore
 #ifdef NETGEN_TRACE_MEMORY
   std::vector<std::string> MemoryTracer::names{"all"};
   std::vector<int> MemoryTracer::parents{-1};
+  std::atomic<size_t> MemoryTracer::total_memory{0};
 #endif // NETGEN_TRACE_MEMORY
 
 } // namespace ngcore

libsrc/core/python_ngcore.cpp

@@ -14,13 +14,11 @@ namespace ngcore
   {
     if (py::isinstance<py::dict>(value))
       {             
-        py::dict vdd(value);
+        Flags nested_flags;
-        // call recursively to set dictionary
+        for(auto item : value.cast<py::dict>())
-        for (auto item : vdd) {
+          SetFlag(nested_flags, item.first.cast<string>(),
-          string name = item.first.cast<string>();
+                  item.second.cast<py::object>());
-          py::object val = py::reinterpret_borrow<py::object>(item.second);
+        flags.SetFlag(s, nested_flags);
-          SetFlag(flags, name, val);
-        }
         return;
       }
 
@@ -103,7 +101,9 @@ namespace ngcore
           }
       }
 
-    auto flags = py::cast<Flags>(flags_dict);
+    Flags flags;
+    for(auto item : flags_dict)
+      SetFlag(flags, item.first.cast<string>(), item.second.cast<py::object>());
 
     for (auto item : kwargs)
       {

libsrc/core/python_ngcore.hpp

@@ -13,11 +13,17 @@
 #include "archive.hpp"
 #include "flags.hpp"
 #include "ngcore_api.hpp"
-#include "profiler.hpp"
+#include "ng_mpi.hpp"
+
 namespace py = pybind11;
 
 namespace ngcore
 {
+#ifdef PARALLEL
+  NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &);
+  NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm);
+#endif // PARALLEL
+
   namespace detail
   {
     template<typename T>
@@ -31,6 +37,16 @@ namespace ngcore
       static constexpr bool value = decltype(check((T*) nullptr))::value;
     };
   } // namespace detail
+
+#ifdef PARALLEL
+  struct mpi4py_comm {
+    mpi4py_comm() = default;
+    mpi4py_comm(NG_MPI_Comm value) : value(value) {}
+    operator NG_MPI_Comm () { return value; }
+
+    NG_MPI_Comm value;
+  };
+#endif  // PARALLEL
 } // namespace ngcore
 
 
@@ -39,6 +55,27 @@ namespace ngcore
 namespace pybind11 {
 namespace detail {
 
+#ifdef PARALLEL
+template <> struct type_caster<ngcore::mpi4py_comm> {
+  public:
+  PYBIND11_TYPE_CASTER(ngcore::mpi4py_comm, _("mpi4py_comm"));
+
+    // Python -> C++
+    bool load(handle src, bool) {
+      return ngcore::NG_MPI_CommFromMPI4Py(src, value.value);
+    }
+
+    // C++ -> Python
+    static handle cast(ngcore::mpi4py_comm src,
+                       return_value_policy /* policy */,
+                       handle /* parent */)
+    {
+      // Create an mpi4py handle
+      return ngcore::NG_MPI_CommToMPI4Py(src.value);
+    }
+};
+#endif //  PARALLEL
+
 template <typename Type, typename Value> struct ngcore_list_caster {
     using value_conv = make_caster<Value>;
 
@@ -259,7 +296,8 @@ namespace ngcore
   }
 
   template <typename T>
-  py::object makePyTuple (FlatArray<T> ar)
+  // py::object makePyTuple (FlatArray<T> ar)
+  py::object makePyTuple (const BaseArrayObject<T> & ar)
   {
     py::tuple res(ar.Size());
     for (auto i : Range(ar))
@@ -281,8 +319,9 @@ namespace ngcore
         .def ("__getitem__",
               [](TFlat & self, TIND i) -> T&
                              {
-                               static constexpr int base = IndexBASE<TIND>();
+                               // static constexpr int base = IndexBASE<TIND>();
-                               if (i < base || i >= self.Size()+base)
+                               auto reli = i - IndexBASE<TIND>();
+                               if (reli < 0 || reli >= self.Size())
                                  throw py::index_error();
                                return self[i]; 
                              },
@@ -290,8 +329,9 @@ namespace ngcore
         .def ("__setitem__",
               [](TFlat & self, TIND i, T val) -> T&
                              {
-                               static constexpr int base = IndexBASE<TIND>();
+                               // static constexpr int base = IndexBASE<TIND>();
-                               if (i < base || i >= self.Size()+base)
+                               auto reli = i - IndexBASE<TIND>();                               
+                               if (reli < 0 || reli >= self.Size())
                                  throw py::index_error();
                                self[i] = val;
                                return self[i];

libsrc/core/python_ngcore_export.cpp

@@ -1,11 +1,18 @@
 #include "python_ngcore.hpp"
 #include "bitarray.hpp"
 #include "taskmanager.hpp"
+#include "mpi_wrapper.hpp"
 
 using namespace ngcore;
 using namespace std;
 using namespace pybind11::literals;
 
+namespace pybind11 { namespace detail {
+}} // namespace pybind11::detail
+
+
+
+
 PYBIND11_MODULE(pyngcore, m) // NOLINT
 {
   try
@@ -30,6 +37,12 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
 
   ExportTable<int>(m);
 
+  #ifdef PARALLEL
+  py::class_<NG_MPI_Comm> (m, "_NG_MPI_Comm")
+          ;
+  m.def("InitMPI", &InitMPI, py::arg("mpi_library_path")=nullopt);
+  #endif // PARALLEL
+
   py::class_<BitArray, shared_ptr<BitArray>> (m, "BitArray")
     .def(py::init([] (size_t n) { return make_shared<BitArray>(n); }),py::arg("n"))
     .def(py::init([] (const BitArray& a) { return make_shared<BitArray>(a); } ), py::arg("ba"))
@@ -128,20 +141,34 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
 
     .def(py::self | py::self)
     .def(py::self & py::self)
+    #ifdef __clang__ 
+    // see https://github.com/pybind/pybind11/issues/1893
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wself-assign-overloaded"
+    #endif
     .def(py::self |= py::self)
     .def(py::self &= py::self)
+    #ifdef __clang__
+    #pragma GCC diagnostic pop
+    #endif
     .def(~py::self)
     ;
 
   py::class_<Flags>(m, "Flags")
     .def(py::init<>())
     .def("__str__", &ToString<Flags>)
-    .def(py::init([](py::object & obj) {
+    .def(py::init([](py::dict kwargs) {
           Flags flags;
-          py::dict d(obj);          
+          for (auto d : kwargs)
-          SetFlag (flags, "", d);
+            SetFlag(flags, d.first.cast<string>(), d.second.cast<py::object>());
           return flags;
-        }), py::arg("obj"), "Create Flags by given object")
+    }), "Create flags from dict")
+    .def(py::init([](py::kwargs kwargs) {
+          Flags flags;
+          for (auto d : kwargs)
+            SetFlag(flags, d.first.cast<string>(), d.second.cast<py::object>());
+          return flags;
+        }), "Create flags from kwargs")
     .def(py::pickle([] (const Flags& self)
         {
           std::stringstream str;
@@ -195,6 +222,10 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
     {
       return CreateDictFromFlags(flags);
     })
+    .def("items", [](const Flags& flags)
+    {
+      return CreateDictFromFlags(flags).attr("items")();
+    })
   ;
   py::implicitly_convertible<py::dict, Flags>();
 
@@ -293,6 +324,8 @@ threads : int
 #endif // NETGEN_TRACE_MEMORY
     ;
 
+    m.def("GetTotalMemory", MemoryTracer::GetTotalMemory);
+
     py::class_<Timer<>> (m, "Timer")
     .def(py::init<const string&>())
     .def("Start", static_cast<void (Timer<>::*)()const>(&Timer<>::Start), "start timer")
@@ -324,4 +357,37 @@ threads : int
 	   }, "Returns list of timers"
 	   );
   m.def("ResetTimers", &NgProfiler::Reset);
+
+  py::class_<NgMPI_Comm> (m, "MPI_Comm")
+#ifdef PARALLEL
+    .def(py::init([] (mpi4py_comm comm) { return NgMPI_Comm(comm); }))
+    .def("WTime", [](NgMPI_Comm  & c) { return NG_MPI_Wtime(); })
+    .def_property_readonly ("mpi4py", [](NgMPI_Comm & self) { return NG_MPI_CommToMPI4Py(self); })
+#endif  // PARALLEL
+    .def_property_readonly ("rank", &NgMPI_Comm::Rank)
+    .def_property_readonly ("size", &NgMPI_Comm::Size)
+    .def("Barrier", &NgMPI_Comm::Barrier)
+    .def("Sum", [](NgMPI_Comm  & c, double x) { return c.AllReduce(x, NG_MPI_SUM); })
+    .def("Min", [](NgMPI_Comm  & c, double x) { return c.AllReduce(x, NG_MPI_MIN); })
+    .def("Max", [](NgMPI_Comm  & c, double x) { return c.AllReduce(x, NG_MPI_MAX); })
+    .def("Sum", [](NgMPI_Comm  & c, int x) { return c.AllReduce(x, NG_MPI_SUM); })
+    .def("Min", [](NgMPI_Comm  & c, int x) { return c.AllReduce(x, NG_MPI_MIN); })
+    .def("Max", [](NgMPI_Comm  & c, int x) { return c.AllReduce(x, NG_MPI_MAX); })
+    .def("Sum", [](NgMPI_Comm  & c, size_t x) { return c.AllReduce(x, NG_MPI_SUM); })
+    .def("Min", [](NgMPI_Comm  & c, size_t x) { return c.AllReduce(x, NG_MPI_MIN); })
+    .def("Max", [](NgMPI_Comm  & c, size_t x) { return c.AllReduce(x, NG_MPI_MAX); })
+    .def("SubComm", [](NgMPI_Comm & c, std::vector<int> proc_list) {
+        Array<int> procs(proc_list.size());
+        for (int i = 0; i < procs.Size(); i++)
+          { procs[i] = proc_list[i]; }
+        if (!procs.Contains(c.Rank()))
+          { throw Exception("rank "+ToString(c.Rank())+" not in subcomm"); }
+	return c.SubCommunicator(procs);
+      }, py::arg("procs"));
+  ;
+
+    
+#ifdef PARALLEL
+  py::implicitly_convertible<mpi4py_comm, NgMPI_Comm>();
+#endif // PARALLEL
 }

libsrc/core/register_archive.hpp

@@ -34,6 +34,8 @@ namespace ngcore {
     return *this;
   }
 
+  /*
+    // now using has_shared_from_this2 in archive.hpp
   template <typename T>
   struct has_shared_from_this
   {
@@ -42,6 +44,7 @@ namespace ngcore {
     typedef decltype( check<T>(sizeof(char)) ) type;
     static constexpr type value = type();
   };
+  */
 #endif // NETGEN_PYTHON
 
 
@@ -59,7 +62,7 @@ namespace ngcore {
       {
         detail::TCargs<T> args;
         ar &args;
-        auto nT = detail::constructIfPossible<T>(args);
+        auto nT = detail::constructIfPossible<T>(std::move(args));
         return typeid(T) == ti ? nT
           : Archive::Caster<T, Bases>::tryUpcast(ti, nT);
       };
@@ -73,8 +76,8 @@ namespace ngcore {
       };
 #ifdef NETGEN_PYTHON
     info.anyToPyCaster = [](const std::any &a) {
-      if constexpr(has_shared_from_this<T>::value) {
+      if constexpr(has_shared_from_this2<T>::value) {
-        std::shared_ptr<T> val = std::any_cast<std::shared_ptr<T>>(&a);
+        std::shared_ptr<T> val = std::any_cast<std::shared_ptr<T>>(a);
         return pybind11::cast(val);
       } else {
         const T* val = std::any_cast<T>(&a);

libsrc/core/signal.hpp

@@ -2,6 +2,7 @@
 #define NGCORE_SIGNALS_HPP
 
 #include <list>
+#include <map>
 #include <functional>
 
 namespace ngcore
@@ -43,6 +44,39 @@ namespace ngcore
     }
     inline bool GetEmitting() const { return is_emitting; }
   };
+
+
+
+
+  class SimpleSignal
+  {
+  private:
+    // std::map<void*,std::function<void()>> funcs;
+    std::list<std::pair<void*,std::function<void()>>> funcs;
+  public:
+    SimpleSignal() = default;
+
+    template<typename FUNC>
+    void Connect(void* var, FUNC f)
+    {
+      // funcs[var] = f;
+      funcs.push_back ( { var, f } );
+    }
+
+    void Remove(void* var)
+    {
+      // funcs.erase(var);
+      funcs.remove_if([&] (auto var_f) { return var_f.first==var; });
+    }
+
+    inline void Emit()
+    {
+      for (auto [key,f] : funcs)
+        f();
+    }
+  };
+
+  
 } // namespace ngcore
 
 #endif // NGCORE_SIGNALS_HPP

libsrc/core/simd_avx512.hpp

@@ -61,6 +61,7 @@ namespace ngcore
 
 
     NETGEN_INLINE auto operator[] (int i) const { return ((int64_t*)(&data))[i]; }
+    NETGEN_INLINE auto & operator[] (int i) { return ((int64_t*)(&data))[i]; }
     NETGEN_INLINE __m512i Data() const { return data; }
     NETGEN_INLINE __m512i & Data() { return data; }
     static SIMD FirstInt() { return { 0, 1, 2, 3, 4, 5, 6, 7 }; }
@@ -111,7 +112,7 @@ namespace ngcore
     template <typename Function>
     void SIMD_function (const Function & func, std::true_type)
     {
-      data = (__m512){ func(7), func(6), func(5), func(4),
+      data = (__m512d){ func(7), func(6), func(5), func(4),
                        func(3), func(2), func(1), func(0) };
     }
 
@@ -132,6 +133,7 @@ namespace ngcore
     }
 
     NETGEN_INLINE double operator[] (int i) const { return ((double*)(&data))[i]; }
+    NETGEN_INLINE double & operator[] (int i) { return ((double*)(&data))[i]; }
     NETGEN_INLINE __m512d Data() const { return data; }
     NETGEN_INLINE __m512d & Data() { return data; }
 

libsrc/core/symboltable.hpp

@@ -45,7 +45,7 @@ namespace ngcore
     }
 
     /// INDEX of symbol name, throws exception if unused
-    size_t Index (const std::string & name) const
+    size_t Index (std::string_view name) const
     {
       for (size_t i = 0; i < names.size(); i++)
         if (names[i] == name) return i;
@@ -53,7 +53,7 @@ namespace ngcore
     }
 
     /// Index of symbol name, returns -1 if unused
-    int CheckIndex (const std::string & name) const
+    int CheckIndex (std::string_view name) const
     {
       for (int i = 0; i < names.size(); i++)
         if (names[i] == name) return i;
@@ -67,12 +67,12 @@ namespace ngcore
     }
 
     /// Returns reference to element. exception for unused identifier
-    reference operator[] (const std::string & name)
+    reference operator[] (std::string_view name)
     {
       return data[Index (name)];
     }
 
-    const_reference operator[] (const std::string & name) const
+    const_reference operator[] (std::string_view name) const
     {
       return data[Index (name)];
     }
@@ -99,7 +99,7 @@ namespace ngcore
     }
 
     /// Associates el to the string name, overrides if name is used
-    void Set (const std::string & name, const T & el)
+    void Set (std::string_view name, const T & el)
     {
       int i = CheckIndex (name);
       if (i >= 0)
@@ -107,14 +107,23 @@ namespace ngcore
       else
         {
           data.push_back(el);
-          names.push_back(name);
+          names.push_back(std::string(name));
         }
     }
 
+
+    
+    /*
     bool Used (const std::string & name) const
     {
       return CheckIndex(name) >= 0;
     }
+    */
+    
+    bool Used (std::string_view name) const
+    {
+      return CheckIndex(name) >= 0;
+    }
     
     /// Deletes symboltable
     inline void DeleteAll ()

libsrc/core/table.cpp

@@ -62,9 +62,9 @@ namespace ngcore
     return index;
   }
 
-  NGCORE_API size_t * TablePrefixSum32 (FlatArray<unsigned int> entrysize)
+  NGCORE_API size_t * TablePrefixSum32 (FlatArray<uint32_t> entrysize)
   { return TablePrefixSum2 (entrysize); }
-  NGCORE_API size_t * TablePrefixSum64 (FlatArray<size_t> entrysize)
+  NGCORE_API size_t * TablePrefixSum64 (FlatArray<uint64_t> entrysize)
   { return TablePrefixSum2 (entrysize); }
 
   /*

libsrc/core/table.hpp

@@ -17,6 +17,7 @@
 #include "ngcore_api.hpp"
 #include "profiler.hpp"
 
+
 namespace ngcore
 {
 
@@ -93,6 +94,7 @@ namespace ngcore
     Iterator end() const { return Iterator(*this, BASE+size); }
   };
 
+  /*
   NGCORE_API extern size_t * TablePrefixSum32 (FlatArray<unsigned int> entrysize);
   NGCORE_API extern size_t * TablePrefixSum64 (FlatArray<size_t> entrysize);
 
@@ -105,6 +107,19 @@ namespace ngcore
   { return TablePrefixSum32 (FlatArray<unsigned> (entrysize.Size(), (unsigned int*)(std::atomic<int>*)entrysize.Addr(0))); }
   NETGEN_INLINE size_t * TablePrefixSum (FlatArray<size_t> entrysize)
   { return TablePrefixSum64 (entrysize); }
+  */
+
+  NGCORE_API extern size_t * TablePrefixSum32 (FlatArray<uint32_t> entrysize);
+  NGCORE_API extern size_t * TablePrefixSum64 (FlatArray<uint64_t> entrysize);
+
+  template <typename T> // TODO: enable_if T is integral
+  NETGEN_INLINE size_t * TablePrefixSum (FlatArray<T> entrysize)
+  {
+    if constexpr (sizeof(T) == 4)
+      return TablePrefixSum32 ( { entrysize.Size(), (uint32_t*)(void*)entrysize.Addr(0) });
+    else
+      return TablePrefixSum64 ( { entrysize.Size(), (uint64_t*)(void*)entrysize.Addr(0) });
+  }
   
 
   /**
@@ -130,6 +145,7 @@ namespace ngcore
     {
       for (size_t i : IntRange(size+1))
         index[i] = i*entrysize;
+      mt.Alloc(GetMemUsage());
     }
 
     /// Construct table of variable entrysize
@@ -141,6 +157,7 @@ namespace ngcore
       index = TablePrefixSum (FlatArray<TI> (entrysize.Size(), entrysize.Data()));
       size_t cnt = index[size];
       data = new T[cnt];
+      mt.Alloc(GetMemUsage());
     }
 
     explicit NETGEN_INLINE Table (const FlatTable<T,IndexType> & tab2)
@@ -157,6 +174,7 @@ namespace ngcore
       size_t cnt = index[size];
       data = new T[cnt];
       this->AsArray() = tab2.AsArray();
+      mt.Alloc(GetMemUsage());
       /*
       for (size_t i = 0; i < cnt; i++)
         data[i] = tab2.data[i];
@@ -177,12 +195,14 @@ namespace ngcore
       data = new T[cnt];
       for (size_t i = 0; i < cnt; i++)
         data[i] = tab2.data[i];
+
+      mt.Alloc(GetMemUsage());
     }
 
     NETGEN_INLINE Table (Table && tab2)
       : FlatTable<T,IndexType>(0, nullptr, nullptr)
     {
-      tab2.mt.Free(tab2.GetMemUsage());
+      mt = std::move(tab2.mt);
       Swap (size, tab2.size);
       Swap (index, tab2.index);
       Swap (data, tab2.data);
@@ -210,7 +230,7 @@ namespace ngcore
 
     NETGEN_INLINE Table & operator= (Table && tab2)
     {
-      mt.Swap(GetMemUsage(), tab2.mt, tab2.GetMemUsage());
+      mt = std::move(tab2.mt);
       Swap (size, tab2.size);
       Swap (index, tab2.index);
       Swap (data, tab2.data);
@@ -324,8 +344,8 @@ namespace ngcore
 	case 1:
           {
             size_t oldval = nd;
-            while (blocknr+1>nd) {
+            while (blocknr-IndexBASE<IndexType>()+1>nd) {
-              nd.compare_exchange_weak (oldval, blocknr+1);
+              nd.compare_exchange_weak (oldval, blocknr-IndexBASE<IndexType>()+1);
               oldval = nd;
             }
             break;
@@ -449,6 +469,8 @@ namespace ngcore
 
   
 
+  
+
   /**
      A dynamic table class.
 

libsrc/core/taskmanager.cpp

@@ -168,6 +168,12 @@ namespace ngcore
         trace = nullptr;
       }
     num_threads = 1;
+#ifdef USE_NUMA
+      for (int j = 0; j < num_nodes; j++)
+          numa_free (nodedata[j], sizeof(NodeData));
+#else
+      delete nodedata[0];
+#endif
   }
 
 #ifdef WIN32

libsrc/core/taskmanager.hpp

@@ -10,6 +10,7 @@
 #include <atomic>
 #include <functional>
 #include <list>
+#include <cmath>
 #include <ostream>
 #include <thread>
 
@@ -316,6 +317,7 @@ namespace ngcore
     
   public:
     SharedLoop (IntRange ar) : r(ar) { cnt = r.begin(); }
+    SharedLoop (size_t s) : SharedLoop (IntRange{s}) { ; }
     SharedIterator begin() { return SharedIterator (cnt, r.end(), true); }
     SharedIterator end()   { return SharedIterator (cnt, r.end(), false); }
   };
@@ -622,6 +624,8 @@ public:
       Reset (r);
     }
 
+    SharedLoop2 (size_t s) : SharedLoop2 (IntRange{s}) { } 
+    
     void Reset (IntRange r)
     {
       for (size_t i = 0; i < ranges.Size(); i++)
@@ -632,6 +636,9 @@ public:
       processed.store(0, std::memory_order_release);
     }
 
+    void Reset (size_t s) { Reset(IntRange{s}); }
+      
+    
     SharedIterator begin()
     {
       /*
@@ -1010,7 +1017,7 @@ public:
     int num_nodes = numa_num_configured_nodes();
     size_t pagesize = numa_pagesize();
     
-    int npages = ceil ( double(s)*sizeof(T) / pagesize );
+    int npages = std::ceil ( double(s)*sizeof(T) / pagesize );
 
     // cout << "size = " << numa_size << endl;
     // cout << "npages = " << npages << endl;

libsrc/core/utils.cpp

@@ -3,16 +3,25 @@
 #include "logging.hpp"
 #include "simd_generic.hpp"
 
-#ifndef WIN32
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#undef WIN32_LEAN_AND_MEAN
+#else // WIN32
 #include <cxxabi.h>
-#endif
+#include <dlfcn.h>
+#endif //WIN32
+//
 #include <array>
 #include <filesystem>
 #include <iostream>
 #include <regex>
+#include <string>
+#include <thread>
 
 #include "ngstream.hpp"
 
+
 namespace ngcore
 {
     namespace detail
@@ -109,7 +118,7 @@ namespace ngcore
 
   const std::chrono::time_point<TClock> wall_time_start = TClock::now();
 
-  int printmessage_importance = 0;
+  int printmessage_importance = getenv("NG_MESSAGE_LEVEL") ? atoi(getenv("NG_MESSAGE_LEVEL")) : 0;
   bool NGSOStream :: glob_active = true;
 
   NGCORE_API int GetCompiledSIMDSize()
@@ -134,5 +143,91 @@ namespace ngcore
       return path;
   }
 
+
+  SharedLibrary :: SharedLibrary(const std::filesystem::path & lib_name_, std::optional<std::filesystem::path> directory_to_delete_, bool global )
+      : lib_name(lib_name_),directory_to_delete(directory_to_delete_)
+  {
+    Load(lib_name, global);
+  }
+
+  SharedLibrary :: ~SharedLibrary()
+  {
+    Unload();
+    if(directory_to_delete)
+      for([[maybe_unused]] auto i : Range(5))
+      {
+        // on Windows, a (detached?) child process of the compiler/linker might still block the directory
+        // wait for it to finish (up to a second)
+        try
+        {
+          std::filesystem::remove_all(*directory_to_delete);
+          directory_to_delete = std::nullopt;
+          break;
+        }
+        catch(const std::exception &e)
+        {
+          std::this_thread::sleep_for(std::chrono::milliseconds(200));
+        }
+      }
+    if(directory_to_delete)
+      std::cerr << "Could not delete " << directory_to_delete->string() << std::endl;
+  }
+
+  void SharedLibrary :: Load( const std::filesystem::path & lib_name_, bool global )
+  {
+    Unload();
+    lib_name = lib_name_;
+#ifdef WIN32
+    lib = LoadLibraryW(lib_name.wstring().c_str());
+    if (!lib) throw std::runtime_error(std::string("Could not load library ") + lib_name.string());
+#else // WIN32
+    auto flags = RTLD_NOW;
+    if (global) flags |= RTLD_GLOBAL;
+    lib = dlopen(lib_name.c_str(), flags);
+    if(lib == nullptr) throw std::runtime_error(dlerror());
+#endif // WIN32
+  }
+
+  void SharedLibrary :: Unload() {
+    if(lib)
+    {
+#ifdef WIN32
+      FreeLibrary((HMODULE)lib);
+#else // WIN32
+      int rc = dlclose(lib);
+      if(rc != 0) std::cerr << "Failed to close library " << lib_name << std::endl;
+#endif // WIN32
+    }
+  }
+
+  void* SharedLibrary :: GetRawSymbol( std::string func_name )
+  {
+#ifdef WIN32
+    void* func = GetProcAddress((HMODULE)lib, func_name.c_str());
+    if(func == nullptr)
+      throw std::runtime_error(std::string("Could not find function ") + func_name + " in library " + lib_name.string());
+#else // WIN32
+    void* func = dlsym(lib, func_name.c_str());
+    if(func == nullptr)
+        throw std::runtime_error(dlerror());
+#endif // WIN32
+
+    return func;
+  }
+
+  void* GetRawSymbol( std::string func_name )
+  {
+    void * func = nullptr;
+#ifdef WIN32
+    throw std::runtime_error("GetRawSymbol not implemented on WIN32");
+#else // WIN32
+    func = dlsym(RTLD_DEFAULT, func_name.c_str());
+    if(func == nullptr)
+        throw std::runtime_error(dlerror());
+#endif // WIN32
+    return func;
+  }
+
+
 } // namespace ngcore
 

libsrc/core/utils.hpp

@@ -6,6 +6,7 @@
 #include <filesystem>
 #include <map>
 #include <ostream>
+#include <optional>
 #include <sstream>
 #include <string>
 
@@ -181,7 +182,7 @@ namespace ngcore
   
   /// square element 
   template <class T>
-  NETGEN_INLINE T sqr (const T a)
+  NETGEN_INLINE constexpr T sqr (const T a)
   {
     return a * a; 
   }
@@ -347,6 +348,41 @@ namespace ngcore
 
   NGCORE_API std::filesystem::path GetTempFilename();
 
+  NGCORE_API void* GetRawSymbol( std::string func_name );
+
+  template <typename TFunc>
+  TFunc GetSymbol( std::string func_name )
+  {
+    return reinterpret_cast<TFunc>(GetRawSymbol(func_name));
+  }
+
+  // Class to handle/load shared libraries
+  class NGCORE_API SharedLibrary
+  {
+    std::filesystem::path lib_name;
+    std::optional<std::filesystem::path> directory_to_delete = std::nullopt;
+    void *lib = nullptr;
+
+  public:
+    SharedLibrary() = default;
+    SharedLibrary(const std::filesystem::path & lib_name_, std::optional<std::filesystem::path> directory_to_delete_ = std::nullopt, bool global = false );
+
+    SharedLibrary(const SharedLibrary &) = delete;
+    SharedLibrary & operator =(const SharedLibrary &) = delete;
+
+    ~SharedLibrary();
+
+    template <typename TFunc>
+    TFunc GetSymbol( std::string func_name )
+    {
+      return reinterpret_cast<TFunc>(GetRawSymbol(func_name));
+    }
+
+    void Load( const std::filesystem::path & lib_name_, bool global = true);
+    void Unload();
+    void* GetRawSymbol( std::string func_name );
+  };
+
 } // namespace ngcore
 
 #endif // NETGEN_CORE_UTILS_HPP

libsrc/csg/csgeom.cpp

@@ -1211,7 +1211,7 @@ namespace netgen
 	PrintMessage (2, "Object ", i, " has ", tams->GetNT(), " triangles");
       }
       }
-    catch (exception)
+    catch (const std::exception &)
       {
 	cerr << "*************************************************************" << endl
 	     << "****   out of memory problem in CSG visualization        ****" << endl

libsrc/csg/edgeflw.cpp

@@ -567,7 +567,7 @@ namespace netgen
     {
     //    int i, j;
     SegmentIndex si;
-    PointIndex pi;
+    // PointIndex pi;
 
     NgArray<int> osedges(cntedge);
     INDEX_2_HASHTABLE<int> osedgesht (cntedge+1);
@@ -610,7 +610,7 @@ namespace netgen
     for (int i = 1; i <= osedgesht.GetNBags(); i++)
       for (int j = 1; j <= osedgesht.GetBagSize(i); j++)
 	{
-	  INDEX_2 i2; 
+	  PointIndices<2> i2; 
 	  int val;
 	  osedgesht.GetData (i, j, i2, val);
 
@@ -619,8 +619,8 @@ namespace netgen
 	  Vec<3> v = p2 - p1;
 	  double vlen = v.Length();
 	  v /= vlen;
-	  for (pi = PointIndex::BASE; 
+	  for (PointIndex pi = IndexBASE<PointIndex>(); 
-	       pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	       pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 
 	    if (pi != i2.I1() && pi != i2.I2())
 	      {
@@ -1371,8 +1371,8 @@ namespace netgen
     lastpi = PointIndex::INVALID;
 
     /*
-    for (pi = PointIndex::BASE; 
+    for (pi = IndexBASE<PointIndex>(); 
-	 pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	 pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
       if (Dist (mesh[pi], p) < 1e-6)
 	{
 	  lastpi = pi;
@@ -1414,8 +1414,8 @@ namespace netgen
 	if (i == ne)
 	  {
 	    /*
-	  for (pi = PointIndex::BASE; 
+	  for (pi = IndexBASE<PointIndex>(); 
-	       pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	       pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 	    if (Dist(mesh[pi], np) < 1e-6)
 	      thispi = pi;
 	    */
@@ -1539,8 +1539,8 @@ namespace netgen
     // generate initial point
     Point<3> p = edgepoints[0];
     PointIndex pi1 = PointIndex::INVALID;
-    for (pi = PointIndex::BASE; 
+    for (PointIndex pi = IndexBASE<PointIndex>(); 
-	 pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	 pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 
       if (Dist (mesh[pi], p) < 1e-6*geometry.MaxSize())
 	{
@@ -1557,8 +1557,8 @@ namespace netgen
 
     p = edgepoints.Last();
     PointIndex pi2 = PointIndex::INVALID;
-    for (pi = PointIndex::BASE; 
+    for (pi = IndexBASE<PointIndex>(); 
-	 pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	 pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 
       if (Dist (mesh[pi], p) < 1e-6*geometry.MaxSize())
 	{
@@ -1646,7 +1646,7 @@ namespace netgen
 	    Mesh & mesh)
   {
     int k;
-    PointIndex pi;
+    // PointIndex pi;
 
     double size = geometry.MaxSize();
     
@@ -1660,8 +1660,8 @@ namespace netgen
       
 	PointIndex frompi = PointIndex::INVALID;
 	PointIndex topi = PointIndex::INVALID;
-	for (pi = PointIndex::BASE; 
+	for (PointIndex pi = IndexBASE<PointIndex>(); 
-	     pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	     pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 	  {
 	    if (Dist2 (mesh[pi], fromp) <= 1e-16*size)
 	      frompi = pi;
@@ -1714,12 +1714,12 @@ namespace netgen
 	if (oldseg.seginfo == 0)
 	  continue;
 
-	int pi1 = oldseg[0];
+	PointIndex pi1 = oldseg[0];
-	int pi2 = oldseg[1];
+	PointIndex pi2 = oldseg[1];
 
-	int npi1 = geometry.identifications.Get(copyedgeidentification)
+	PointIndex npi1 = geometry.identifications.Get(copyedgeidentification)
 	  -> GetIdentifiedPoint (mesh, pi1);
-	int npi2 = geometry.identifications.Get(copyedgeidentification)
+	PointIndex npi2 = geometry.identifications.Get(copyedgeidentification)
 	  -> GetIdentifiedPoint (mesh, pi2);
 
 	//(*testout) << "copy edge, pts = " << npi1 << " - " << npi2 << endl;
@@ -1885,12 +1885,10 @@ namespace netgen
 
 	    if (seg1.domin != -1 || seg1.domout != -1)
 	      {
-		mesh.AddPoint (p1, layer, EDGEPOINT);
+                seg1[0] = mesh.AddPoint (p1, layer, EDGEPOINT);
-		mesh.AddPoint (p2, layer, EDGEPOINT);
+                seg1[1] = mesh.AddPoint (p2, layer, EDGEPOINT);
-		seg1[0] = mesh.GetNP()-1;
+                seg2[0] = seg1[1];
-		seg1[1] = mesh.GetNP();
+                seg2[1] = seg1[0];
-		seg2[1] = mesh.GetNP()-1;
-		seg2[0] = mesh.GetNP();
 		seg1.geominfo[0].trignum = 1;
 		seg1.geominfo[1].trignum = 1;
 		seg2.geominfo[0].trignum = 1;

libsrc/csg/genmesh.cpp

@@ -35,7 +35,7 @@ namespace netgen
         auto up = geom.GetUserPoint(i);
 	auto pnum = mesh.AddPoint(up);
 	mesh.Points().Last().Singularity (geom.GetUserPointRefFactor(i));
-	mesh.AddLockedPoint (PointIndex (i+1));
+	mesh.AddLockedPoint (pnum);
         int index = up.GetIndex();
         if (index == -1)
           index = mesh.AddCD3Name (up.GetName())+1;
@@ -443,7 +443,7 @@ namespace netgen
 	meshing.SetStartTime (starttime);
 
         double eps = 1e-8 * geom.MaxSize();
-	for (PointIndex pi = PointIndex::BASE; pi < noldp+PointIndex::BASE; pi++)
+	for (PointIndex pi = IndexBASE<PointIndex>(); pi < noldp+IndexBASE<PointIndex>(); pi++)
 	  { 
 	    // if(surf->PointOnSurface(mesh[pi]))
 	    meshing.AddPoint (mesh[pi], pi, NULL,
@@ -473,8 +473,8 @@ namespace netgen
 	  {
 	    PointGeomInfo gi;
 	    gi.trignum = k;
-	    meshing.AddBoundaryElement (segments[si][0] + 1 - PointIndex::BASE, 
+	    meshing.AddBoundaryElement (segments[si][0] + 1 - IndexBASE<PointIndex>(), 
-					segments[si][1] + 1 - PointIndex::BASE, 
+					segments[si][1] + 1 - IndexBASE<PointIndex>(), 
 					gi, gi);
 	  }
 
@@ -718,7 +718,7 @@ namespace netgen
 
 	    mesh -> LoadLocalMeshSize (mparam.meshsizefilename);
             for (auto mspnt : mparam.meshsize_points)
-              mesh -> RestrictLocalH (mspnt.pnt, mspnt.h);
+              mesh -> RestrictLocalH (mspnt.pnt, mspnt.h, mspnt.layer);
 	  }
 
 	spoints.SetSize(0);
@@ -826,6 +826,9 @@ namespace netgen
       {
 	multithread.task = "Volume meshing";
 
+	for (int i = 0; i < geom.GetNTopLevelObjects(); i++)
+	  mesh->SetMaterial (i+1, geom.GetTopLevelObject(i)->GetMaterial().c_str());
+
 	MESHING3_RESULT res =
 	  MeshVolume (mparam, *mesh);
 
@@ -838,10 +841,6 @@ namespace netgen
 
 	MeshQuality3d (*mesh);
       
-	for (int i = 0; i < geom.GetNTopLevelObjects(); i++)
-	  mesh->SetMaterial (i+1, geom.GetTopLevelObject(i)->GetMaterial().c_str());
-      
-
 #ifdef STAT_STREAM
 	(*statout) << GetTime() << " & ";
 #endif      

libsrc/csg/identify.cpp

@@ -65,10 +65,10 @@ ShortEdge (const SpecialPoint & sp1, const SpecialPoint & sp2) const
   return 0;
 }
 
-int Identification :: GetIdentifiedPoint (class Mesh & mesh, int pi)
+PointIndex Identification :: GetIdentifiedPoint (class Mesh & mesh, PointIndex pi)
 {
   cout << "Identification::GetIdentifiedPoint called for base-class" << endl;
-  return -1;
+  return PointIndex::INVALID;
 }
 
 void Identification :: IdentifyPoints (Mesh & mesh)
@@ -261,8 +261,8 @@ Identifiable (const Point<3> & p1, const Point<3> & p2) const
 
 
 
-int PeriodicIdentification :: 
+PointIndex PeriodicIdentification :: 
-GetIdentifiedPoint (class Mesh & mesh,  int pi)
+GetIdentifiedPoint (class Mesh & mesh, PointIndex pi)
 {
   const Surface *snew;
   const Point<3> & p = mesh.Point (pi);
@@ -289,14 +289,14 @@ GetIdentifiedPoint (class Mesh & mesh,  int pi)
   // project to other surface
   snew->Project (hp);
 
-  int newpi = 0;
+  PointIndex newpi(PointIndex::INVALID);
-  for (int i = 1; i <= mesh.GetNP(); i++)
+  for (PointIndex pi : Range(mesh.Points()))
-    if (Dist2 (mesh.Point(i), hp) < 1e-12)
+    if (Dist2 (mesh.Point(pi), hp) < 1e-12)
       {
-	newpi = i;
+	newpi = pi;
 	break;
       }
-  if (!newpi)
+  if (!newpi.IsValid())
     newpi = mesh.AddPoint (hp);
 
   if (snew == s2)
@@ -322,6 +322,7 @@ void PeriodicIdentification :: IdentifyPoints (class Mesh & mesh)
   mesh.GetBox(p1, p2);
   auto eps = 1e-6 * (p2-p1).Length();
 
+  /*
   for (int i = 1; i <= mesh.GetNP(); i++)
     {
       Point<3> p = mesh.Point(i);
@@ -334,13 +335,24 @@ void PeriodicIdentification :: IdentifyPoints (class Mesh & mesh)
 	    if (Dist2(mesh.Point(j), pp) < eps)
 	      {
 		mesh.GetIdentifications().Add (i, j, nr);
-		/*
-		(*testout) << "Identify points(periodic:), nr = " << nr << ": "
-			   << mesh.Point(i) << " - " << mesh.Point(j) << endl;
-		*/
 	      }
 	}
     }
+  */
+
+  for (auto pi : Range(mesh.Points()))
+    {
+      Point<3> p = mesh[pi];
+      if (s1->PointOnSurface (p))
+	{
+	  Point<3> pp = p;
+          pp = trafo(pp);
+	  s2->Project (pp);
+          for (PointIndex pj : Range(mesh.Points()))
+	    if (Dist2(mesh[pj], pp) < eps)
+              mesh.GetIdentifications().Add (pi, pj, nr);
+	}
+    }
 
   mesh.GetIdentifications().SetType(nr,Identifications::PERIODIC);
 }
@@ -396,15 +408,15 @@ void PeriodicIdentification :: IdentifyFaces (class Mesh & mesh)
 
 		    if (side == 1)
 		      {
-			if (mesh.GetIdentifications().Get (seg1[0], seg2[0]) &&
+			if (mesh.GetIdentifications().Used (seg1[0], seg2[0]) &&
-			    mesh.GetIdentifications().Get (seg1[1], seg2[1]))
+			    mesh.GetIdentifications().Used (seg1[1], seg2[1]))
 			  {
 			    foundother = 1;
 			    break;
 			  }
 			
-			if (mesh.GetIdentifications().Get (seg1[0], seg2[1]) &&
+			if (mesh.GetIdentifications().Used (seg1[0], seg2[1]) &&
-			    mesh.GetIdentifications().Get (seg1[1], seg2[0]))
+			    mesh.GetIdentifications().Used (seg1[1], seg2[0]))
 			  {
 			    foundother = 1;
 			    break;
@@ -412,15 +424,15 @@ void PeriodicIdentification :: IdentifyFaces (class Mesh & mesh)
 		      }
 		    else
 		      {
-			if (mesh.GetIdentifications().Get (seg2[0], seg1[0]) &&
+			if (mesh.GetIdentifications().Used (seg2[0], seg1[0]) &&
-			    mesh.GetIdentifications().Get (seg2[1], seg1[1]))
+			    mesh.GetIdentifications().Used (seg2[1], seg1[1]))
 			  {
 			    foundother = 1;
 			    break;
 			  }
 			
-			if (mesh.GetIdentifications().Get (seg2[0], seg1[1]) &&
+			if (mesh.GetIdentifications().Used (seg2[0], seg1[1]) &&
-			    mesh.GetIdentifications().Get (seg2[1], seg1[0]))
+			    mesh.GetIdentifications().Used (seg2[1], seg1[0]))
 			  {
 			    foundother = 1;
 			    break;
@@ -885,16 +897,20 @@ ShortEdge (const SpecialPoint & sp1, const SpecialPoint & sp2) const
 
 
 
-int CloseSurfaceIdentification :: 
+PointIndex CloseSurfaceIdentification :: 
-GetIdentifiedPoint (class Mesh & mesh,  int pi)
+GetIdentifiedPoint (class Mesh & mesh, PointIndex pi)
 {
   const Surface *snew;
   const Point<3> & p = mesh.Point (pi);
 
-  NgArray<int,PointIndex::BASE> identmap(mesh.GetNP());
+  idmap_type identmap(mesh.GetNP());
   mesh.GetIdentifications().GetMap (nr, identmap);
+  /*
   if (identmap.Get(pi))
     return identmap.Get(pi);
+  */
+  if (identmap[pi].IsValid())
+    return identmap[pi];
   
   
   if (s1->PointOnSurface (p))
@@ -1168,15 +1184,15 @@ void CloseSurfaceIdentification :: IdentifyFaces (class Mesh & mesh)
 		      
 		      if (side == 1)
 			{
-			  if (mesh.GetIdentifications().Get (seg1[0], seg2[0]) &&
+			  if (mesh.GetIdentifications().Used (seg1[0], seg2[0]) &&
-			      mesh.GetIdentifications().Get (seg1[1], seg2[1]))
+			      mesh.GetIdentifications().Used (seg1[1], seg2[1]))
 			    {
 			      foundother = 1;
 			      break;
 			    }
 			  
-			  if (mesh.GetIdentifications().Get (seg1[0], seg2[1]) &&
+			  if (mesh.GetIdentifications().Used (seg1[0], seg2[1]) &&
-			      mesh.GetIdentifications().Get (seg1[1], seg2[0]))
+			      mesh.GetIdentifications().Used (seg1[1], seg2[0]))
 			    {
 			      foundother = 1;
 			      break;
@@ -1184,15 +1200,15 @@ void CloseSurfaceIdentification :: IdentifyFaces (class Mesh & mesh)
 			}
 		      else
 			{
-			  if (mesh.GetIdentifications().Get (seg2[0], seg1[0]) &&
+			  if (mesh.GetIdentifications().Used (seg2[0], seg1[0]) &&
-			      mesh.GetIdentifications().Get (seg2[1], seg1[1]))
+			      mesh.GetIdentifications().Used (seg2[1], seg1[1]))
 			    {
 			      foundother = 1;
 			      break;
 			    }
 			  
-			  if (mesh.GetIdentifications().Get (seg2[0], seg1[1]) &&
+			  if (mesh.GetIdentifications().Used (seg2[0], seg1[1]) &&
-			      mesh.GetIdentifications().Get (seg2[1], seg1[0]))
+			      mesh.GetIdentifications().Used (seg2[1], seg1[0]))
 			    {
 			      foundother = 1;
 			      break;
@@ -1229,7 +1245,7 @@ BuildSurfaceElements (NgArray<Segment> & segs,
   bool found = 0;
   int cntquads = 0;
 
-  NgArray<int,PointIndex::BASE>  identmap;
+  idmap_type identmap;
   identmap = 0;
 
   mesh.GetIdentifications().GetMap (nr, identmap);
@@ -1650,8 +1666,8 @@ BuildSurfaceElements (NgArray<Segment> & segs,
       {
 	const Segment & s1 = segs.Get(i1);
 	const Segment & s2 = segs.Get(i2);
-	if (mesh.GetIdentifications().Get (s1[0], s2[1]) &&
+	if (mesh.GetIdentifications().Used (s1[0], s2[1]) &&
-	    mesh.GetIdentifications().Get (s1[1], s2[0]))
+	    mesh.GetIdentifications().Used (s1[1], s2[0]))
 	  {
 	    Element2d el(QUAD);
 	    el.PNum(1) = s1[0];

libsrc/csg/identify.hpp

@@ -56,7 +56,7 @@ namespace netgen
     virtual void IdentifyFaces (class Mesh & mesh);
 
     /// get point on other surface, add entry in mesh identifications
-    virtual int GetIdentifiedPoint (class Mesh & mesh, int pi1);
+    virtual PointIndex GetIdentifiedPoint (class Mesh & mesh, PointIndex pi1);
 
     /// copy surfaces, or fill rectangles
     virtual void BuildSurfaceElements (NgArray<class Segment> & segs,
@@ -97,7 +97,7 @@ namespace netgen
 			      const TABLE<int> & specpoint2surface) const override;
 
     virtual int Identifiable (const Point<3> & p1, const Point<3> & sp2) const override;
-    virtual int GetIdentifiedPoint (class Mesh & mesh, int pi1) override;
+    virtual PointIndex GetIdentifiedPoint (class Mesh & mesh, PointIndex pi1) override;
     virtual void IdentifyPoints (class Mesh & mesh) override;
     virtual void IdentifyFaces (class Mesh & mesh) override;
     virtual void BuildSurfaceElements (NgArray<class Segment> & segs,
@@ -153,7 +153,7 @@ namespace netgen
     virtual int Identifiable (const Point<3> & p1, const Point<3> & sp2) const;
     virtual int IdentifiableCandidate (const SpecialPoint & sp1) const;
     virtual int ShortEdge (const SpecialPoint & sp1, const SpecialPoint & sp2) const;
-    virtual int GetIdentifiedPoint (class Mesh & mesh, int pi1);
+    virtual PointIndex GetIdentifiedPoint (class Mesh & mesh, PointIndex pi1);
     const Array<double> & GetSlices () const { return slices; }
     virtual void IdentifyPoints (class Mesh & mesh);
     virtual void IdentifyFaces (class Mesh & mesh);

libsrc/csg/python_csg.cpp

@@ -756,10 +756,8 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
            {
              MeshingParameters mp;
              if(pars) mp = *pars;
-             {
-               py::gil_scoped_acquire aq;
              CreateMPfromKwargs(mp, kwargs);
-             }
+             py::gil_scoped_release gil_rel;
              auto mesh = make_shared<Mesh>();
              SetGlobalMesh (mesh);
              mesh->SetGeometry(geo);
@@ -770,8 +768,7 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
                throw Exception("Meshing failed!");
              return mesh;
            }, py::arg("mp") = nullptr,
-       meshingparameter_description.c_str(),
+       meshingparameter_description.c_str())
-    py::call_guard<py::gil_scoped_release>())
     ;
 
   m.def("Save", FunctionPointer 

libsrc/csg/singularref.cpp

@@ -53,7 +53,7 @@ void SingularEdge :: FindPointsOnEdge (class Mesh & mesh)
 
   for (SegmentIndex si = 0; si < mesh.GetNSeg(); si++)
     {
-      INDEX_2 i2 (mesh[si][0], mesh[si][1]);
+      PointIndices<2> i2 (mesh[si][0], mesh[si][1]);
       /*
       
       bool onedge = 1;
@@ -93,8 +93,8 @@ void SingularEdge :: FindPointsOnEdge (class Mesh & mesh)
 	{
 	  segms.Append (i2);
 	  //	  PrintMessage (5, "sing segment ", i2.I1(), " - ", i2.I2());
-	  points.Append (mesh[ PointIndex (i2.I1())]);
+	  points.Append (mesh[i2.I1()]);
-	  points.Append (mesh[ PointIndex (i2.I2())]);
+	  points.Append (mesh[i2.I2()]);
 	  mesh[si].singedge_left = factor;
 	  mesh[si].singedge_right = factor;
 	}	    
@@ -153,8 +153,8 @@ void SingularPoint :: FindPoints (class Mesh & mesh)
   NgArray<int> surfk, surf;
 
 
-  for (PointIndex pi = PointIndex::BASE; 
+  for (PointIndex pi = IndexBASE<PointIndex>(); 
-       pi < mesh.GetNP()+PointIndex::BASE; pi++)
+       pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
     {
       if (mesh[pi].Type() != FIXEDPOINT) continue;
       const Point<3> p = mesh[pi];

libsrc/csg/vscsg.cpp

@@ -472,7 +472,8 @@ namespace netgen
 	box = Box3d (Point3d (0,0,0), Point3d (1,1,1));
       }
   
-    if (zoomall == 2 && ((vispar.centerpoint >= 1 && vispar.centerpoint <= mesh->GetNP()) ||
+    if (zoomall == 2 && ((vispar.centerpoint-IndexBASE<PointIndex>() >= 0 &&
+                          vispar.centerpoint-IndexBASE<PointIndex>() < mesh->GetNP()) ||
 			 vispar.use_center_coords))
       {
 	if (vispar.use_center_coords)

libsrc/csg/zrefine.cpp

@@ -45,9 +45,10 @@ namespace netgen
   void MakePrismsSingEdge (Mesh & mesh, INDEX_2_HASHTABLE<int> & singedges)
   {
     // volume elements
-    for (int i = 1; i <= mesh.GetNE(); i++)
+    // for (int i = 1; i <= mesh.GetNE(); i++)
+    for (ElementIndex ei = 0; ei < mesh.GetNE(); ei++)
       {
-	Element & el = mesh.VolumeElement(i);
+	Element & el = mesh.VolumeElement(ei);
 	if (el.GetType() != TET) continue;
 
 	for (int j = 1; j <= 3; j++)
@@ -76,9 +77,9 @@ namespace netgen
       }
 
     // surface elements
-    for (int i = 1; i <= mesh.GetNSE(); i++)
+    for (SurfaceElementIndex sei = 0; sei < mesh.GetNSE(); sei++)
       {
-	Element2d & el = mesh.SurfaceElement(i);
+	Element2d & el = mesh.SurfaceElement(sei);
 	if (el.GetType() != TRIG) continue;
 
 	for (int j = 1; j <= 3; j++)
@@ -110,18 +111,18 @@ namespace netgen
   */
   void MakePrismsClosePoints (Mesh & mesh)
   {
-    int i, j, k;
+    // int i, j, k;
-    for (i = 1; i <= mesh.GetNE(); i++)
+    for (ElementIndex ei = 0; ei < mesh.GetNE(); ei++)
       {
-	Element & el = mesh.VolumeElement(i);
+	Element & el = mesh.VolumeElement(ei);
 	if (el.GetType() == TET)
 	  {
-	    for (j = 1; j <= 3; j++)
+	    for (int j = 1; j <= 3; j++)
-	      for (k = j+1; k <= 4; k++)
+	      for (int k = j+1; k <= 4; k++)
 		{
 		  INDEX_2 edge(el.PNum(j), el.PNum(k));
 		  edge.Sort();
-		  if (mesh.GetIdentifications().GetSymmetric (el.PNum(j), el.PNum(k)))
+		  if (mesh.GetIdentifications().UsedSymmetric (el.PNum(j), el.PNum(k)))
 		    {
 		      int pi3 = 1, pi4 = 1;
 		      while (pi3 == j || pi3 == k) pi3++;
@@ -145,7 +146,7 @@ namespace netgen
 	  {
 	    // pyramid, base face = 1,2,3,4
 	  
-	    for (j = 0; j <= 1; j++)
+	    for (int j = 0; j <= 1; j++)
 	      {
 		PointIndex pi1 = el.PNum( (j+0) % 4 + 1);
 		PointIndex pi2 = el.PNum( (j+1) % 4 + 1);
@@ -157,8 +158,8 @@ namespace netgen
 		INDEX_2 edge2(pi2, pi3);
 		edge1.Sort();
 		edge2.Sort();
-		if (mesh.GetIdentifications().GetSymmetric (pi1, pi4) &&
+		if (mesh.GetIdentifications().UsedSymmetric (pi1, pi4) &&
-		    mesh.GetIdentifications().GetSymmetric (pi2, pi3))
+		    mesh.GetIdentifications().UsedSymmetric (pi2, pi3))
 		  {
 		    //int p3 = el.PNum(pi3);
 		    //int p4 = el.PNum(pi4);
@@ -175,18 +176,18 @@ namespace netgen
 	  }
       }
   
-    for (i = 1; i <= mesh.GetNSE(); i++)
+    for (SurfaceElementIndex sei = 0; sei < mesh.GetNSE(); sei++)
       {
-	Element2d & el = mesh.SurfaceElement(i);
+	Element2d & el = mesh.SurfaceElement(sei);
 	if (el.GetType() != TRIG) continue;
 
-	for (j = 1; j <= 3; j++)
+	for (int j = 1; j <= 3; j++)
 	  {
-	    k = (j % 3) + 1;
+	    int k = (j % 3) + 1;
 	    INDEX_2 edge(el.PNum(j), el.PNum(k));
 	    edge.Sort();
 
-	    if (mesh.GetIdentifications().GetSymmetric (el.PNum(j), el.PNum(k)))
+	    if (mesh.GetIdentifications().UsedSymmetric (el.PNum(j), el.PNum(k)))
 	      {
 		int pi3 = 6-j-k;
 		int p3 = el.PNum(pi3);
@@ -244,7 +245,7 @@ namespace netgen
   void RefinePrisms (Mesh & mesh, const CSGeometry * geom, 
 		     ZRefinementOptions & opt)
   {
-    int i, j;
+    // int i, j;
     bool found, change;
     int cnt = 0;
 
@@ -261,15 +262,23 @@ namespace netgen
   
     // if (mesh.GetIdentifications().HasIdentifiedPoints())
       {
-        INDEX_2_HASHTABLE<int> & identpts = 
+        auto & identpts =
           mesh.GetIdentifications().GetIdentifiedPoints ();
 
-	for (i = 1; i <= identpts.GetNBags(); i++)
+        /*
-	  for (j = 1; j <= identpts.GetBagSize(i); j++)
+	for (int i = 1; i <= identpts.GetNBags(); i++)
+	  for (int j = 1; j <= identpts.GetBagSize(i); j++)
 	    {
-	      INDEX_2 pair;
+	      INDEX_3 pair;
-	      int idnr;
+	      int dummy;
-	      identpts.GetData(i, j, pair, idnr);
+	      identpts.GetData(i, j, pair, dummy);
+        */
+        for (auto [hash, val] : identpts)\
+          {
+            auto [hash_pts, idnr] = hash;
+            auto [pi1, pi2] = hash_pts;
+            // auto idnr = pair[2];
+            
 	      const CloseSurfaceIdentification * csid = 
 		dynamic_cast<const CloseSurfaceIdentification*> 
 		(geom->identifications.Get(idnr));
@@ -280,17 +289,25 @@ namespace netgen
 		      if (first_id.Test (idnr))
 			{
 			  first_id.Clear(idnr);
+                          /*
 			  ref_uniform.Append (INDEX_3 (pair.I1(), pair.I2(), csid->RefLevels()));
 			  ref_singular.Append (INDEX_3 (pair.I1(), pair.I2(), csid->RefLevels1()));
 			  ref_singular.Append (INDEX_3 (pair.I2(), pair.I1(), csid->RefLevels2()));
+                          */
+			  ref_uniform.Append (INDEX_3 (pi1, pi2, csid->RefLevels()));
+			  ref_singular.Append (INDEX_3 (pi1, pi2, csid->RefLevels1()));
+			  ref_singular.Append (INDEX_3 (pi2, pi1, csid->RefLevels2()));
+                          
 			}
 		    }
 		  else
 		    {   
 		      //const NgArray<double> & slices = csid->GetSlices();
 		      INDEX_4 i4;
-		      i4[0] = pair.I1();
+		      // i4[0] = pair.I1();
-		      i4[1] = pair.I2();
+		      // i4[1] = pair.I2();
+		      i4[0] = pi1; 
+		      i4[1] = pi2; 
 		      i4[2] = idnr;
 		      i4[3] = csid->GetSlices().Size();
 		      ref_slices.Append (i4);
@@ -313,7 +330,7 @@ namespace netgen
 	found = 0;
 	// mark prisms due to close surface flags:
 	int oldsize = ref_uniform.Size();
-	for (i = 1; i <= oldsize; i++)
+	for (int i = 1; i <= oldsize; i++)
 	  {
 	    int pi1 = ref_uniform.Get(i).I1();
 	    int pi2 = ref_uniform.Get(i).I2();
@@ -339,7 +356,7 @@ namespace netgen
 		ref_uniform.Append (INDEX_3(pi2, npi, levels-1));
 	      }
 	  }
-	for (i = 1; i <= ref_singular.Size(); i++)
+	for (int i = 1; i <= ref_singular.Size(); i++)
 	  {
 	    int pi1 = ref_singular.Get(i).I1();
 	    int pi2 = ref_singular.Get(i).I2();
@@ -367,7 +384,7 @@ namespace netgen
 	      }
 	  }
 
-	for (i = 1; i <= ref_slices.Size(); i++)
+	for (int i = 1; i <= ref_slices.Size(); i++)
 	  {
 	    int pi1 = ref_slices.Get(i)[0];
 	    int pi2 = ref_slices.Get(i)[1];
@@ -413,13 +430,13 @@ namespace netgen
 
 
 
-	for (i = 1; i <= mesh.GetNE(); i++)
+	for (ElementIndex ei = 0; ei < mesh.GetNE(); ei++)
 	  {
-	    Element & el = mesh.VolumeElement (i);
+	    Element & el = mesh.VolumeElement (ei);
 	    if (el.GetType() != PRISM)
 	      continue;
 
-	    for (j = 1; j <= 3; j++)
+	    for (int j = 1; j <= 3; j++)
 	      {
 		int pi1 = el.PNum(j);
 		int pi2 = el.PNum(j+3);
@@ -465,14 +482,14 @@ namespace netgen
 	  {
 	    PrintMessage (5, "start loop");
 	    change = 0;
-	    for (i = 1; i <= mesh.GetNE(); i++)
+	    for (ElementIndex ei = 0; ei < mesh.GetNE(); ei++)
 	      {
-		Element & el = mesh.VolumeElement (i);
+		Element & el = mesh.VolumeElement (ei);
 		if (el.GetType() != PRISM)
 		  continue;
 	      
 		bool hasref = 0, hasnonref = 0;
-		for (j = 1; j <= 3; j++)
+		for (int j = 1; j <= 3; j++)
 		  {
 		    int pi1 = el.PNum(j);
 		    int pi2 = el.PNum(j+3);
@@ -491,7 +508,7 @@ namespace netgen
 		  {
 		    //		  cout << "el " << i << " in closure" << endl;
 		    change = 1;
-		    for (j = 1; j <= 3; j++)
+		    for (int j = 1; j <= 3; j++)
 		      {
 			int pi1 = el.PNum(j);
 			int pi2 = el.PNum(j+3);
@@ -518,7 +535,7 @@ namespace netgen
 
 	int oldns = mesh.GetNSeg();
 
-	for (i = 1; i <= oldns; i++)
+	for (int i = 1; i <= oldns; i++)
 	  {
 	    const Segment & el = mesh.LineSegment(i);
 
@@ -572,14 +589,14 @@ namespace netgen
 
 	// do refinement
 	int oldne = mesh.GetNE();
-	for (i = 1; i <= oldne; i++)
+	for (ElementIndex ei = 0; ei < oldne; ei++)
 	  {
-	    Element & el = mesh.VolumeElement (i);
+	    Element & el = mesh.VolumeElement (ei);
 	    if (el.GetNP() != 6)
 	      continue;
 
 	    int npi[3];
-	    for (j = 1; j <= 3; j++)
+	    for (int j = 1; j <= 3; j++)
 	      {
 		int pi1 = el.PNum(j);
 		int pi2 = el.PNum(j+3);
@@ -607,7 +624,7 @@ namespace netgen
 	    if (npi[0])
 	      {
 		Element nel1(6), nel2(6);
-		for (j = 1; j <= 3; j++)
+		for (int j = 1; j <= 3; j++)
 		  {
 		    nel1.PNum(j) = el.PNum(j);
 		    nel1.PNum(j+3) = npi[j-1];
@@ -616,7 +633,7 @@ namespace netgen
 		  }
 		nel1.SetIndex (el.GetIndex());
 		nel2.SetIndex (el.GetIndex());
-		mesh.VolumeElement (i) = nel1;
+		mesh.VolumeElement (ei) = nel1;
 		mesh.AddVolumeElement (nel2);
 	      }
 	  }
@@ -628,15 +645,15 @@ namespace netgen
 	// do surface elements
 	int oldnse = mesh.GetNSE();
 	//      cout << "oldnse = " << oldnse << endl;
-	for (i = 1; i <= oldnse; i++)
+	for (SurfaceElementIndex sei = 0; sei < oldnse; sei++)
 	  {
-	    Element2d & el = mesh.SurfaceElement (i);
+	    Element2d & el = mesh.SurfaceElement (sei);
 	    if (el.GetType() != QUAD)
 	      continue;
 
 	    int index = el.GetIndex();
 	    int npi[2];
-	    for (j = 1; j <= 2; j++)
+	    for (int j = 1; j <= 2; j++)
 	      {
 		int pi1, pi2;
 
@@ -669,7 +686,7 @@ namespace netgen
 	    if (npi[0])
 	      {
 		Element2d nel1(QUAD), nel2(QUAD);
-		for (j = 1; j <= 4; j++)
+		for (int j = 1; j <= 4; j++)
 		  {
 		    nel1.PNum(j) = el.PNum(j);
 		    nel2.PNum(j) = el.PNum(j);
@@ -690,7 +707,7 @@ namespace netgen
 		nel1.SetIndex (el.GetIndex());
 		nel2.SetIndex (el.GetIndex());
 
-		mesh.SurfaceElement (i) = nel1;
+		mesh.SurfaceElement (sei) = nel1;
 		mesh.AddSurfaceElement (nel2);
 
 		int si = mesh.GetFaceDescriptor (index).SurfNr();
@@ -716,9 +733,9 @@ namespace netgen
 
   void CombineSingularPrisms(Mesh& mesh)
   {
-    for(int i = 1; i<=mesh.GetNE(); i++)
+    for(ElementIndex ei = 0; ei < mesh.GetNE(); ei++)
       {
-        Element& el = mesh.VolumeElement(i);
+        Element& el = mesh.VolumeElement(ei);
         if(el.GetType() != PRISM)
           continue;
         if(el.PNum(3) == el.PNum(6))

libsrc/general/CMakeLists.txt

@@ -2,7 +2,6 @@ target_sources(nglib PRIVATE
     gzstream.cpp
     hashtabl.cpp
     mystring.cpp
-    ngarray.cpp
     ngbitarray.cpp
     optmem.cpp
     parthreads.cpp

libsrc/general/gzstream.cpp

@@ -44,7 +44,7 @@ namespace GZSTREAM_NAMESPACE {
 // class gzstreambuf:
 // --------------------------------------
 
-gzstreambuf* gzstreambuf::open( const filesystem::path & name, int open_mode) {
+  gzstreambuf* gzstreambuf::open( const std::filesystem::path & name, int open_mode) {
     if ( is_open())
         return (gzstreambuf*)0;
     mode = open_mode;
@@ -143,7 +143,7 @@ int gzstreambuf::sync() {
 // class gzstreambase:
 // --------------------------------------
 
-gzstreambase::gzstreambase( const filesystem::path & name, int mode) {
+  gzstreambase::gzstreambase( const std::filesystem::path & name, int mode) {
     init( &buf);
     open( name.c_str(), mode);
 }
@@ -152,7 +152,7 @@ gzstreambase::~gzstreambase() {
     buf.close();
 }
 
-void gzstreambase::open( const filesystem::path & name, int open_mode) {
+  void gzstreambase::open( const std::filesystem::path & name, int open_mode) {
     if ( ! buf.open( name.c_str(), open_mode))
         clear( rdstate() | std::ios::badbit);
 }

libsrc/general/gzstream.h

@@ -62,7 +62,7 @@ public:
         // ASSERT: both input & output capabilities will not be used together
     }
     int is_open() { return opened; }
-    gzstreambuf* open( const filesystem::path & name, int open_mode);
+    gzstreambuf* open( const std::filesystem::path & name, int open_mode);
     gzstreambuf* close();
     ~gzstreambuf() { close(); }
     
@@ -76,9 +76,9 @@ protected:
     gzstreambuf buf;
 public:
     gzstreambase() { init(&buf); }
-    gzstreambase( const filesystem::path & name, int open_mode);
+    gzstreambase( const std::filesystem::path & name, int open_mode);
     ~gzstreambase();
-    void open( const filesystem::path & name, int open_mode);
+    void open( const std::filesystem::path & name, int open_mode);
     void close();
     gzstreambuf* rdbuf() { return &buf; }
 };
@@ -92,10 +92,10 @@ public:
 class DLL_HEADER igzstream : public gzstreambase, public std::istream {
 public:
     igzstream() : std::istream( &buf) {} 
-    igzstream( const filesystem::path & name, int open_mode = std::ios::in)
+    igzstream( const std::filesystem::path & name, int open_mode = std::ios::in)
         : gzstreambase( name, open_mode), std::istream( &buf) {}  
     gzstreambuf* rdbuf() { return gzstreambase::rdbuf(); }
-    void open( const filesystem::path & name, int open_mode = std::ios::in) {
+    void open( const std::filesystem::path & name, int open_mode = std::ios::in) {
         gzstreambase::open( name, open_mode);
     }
 };
@@ -103,10 +103,10 @@ public:
 class DLL_HEADER ogzstream : public gzstreambase, public std::ostream {
 public:
     ogzstream() : std::ostream( &buf) {}
-    ogzstream( const filesystem::path & name, int mode = std::ios::out)
+    ogzstream( const std::filesystem::path & name, int mode = std::ios::out)
         : gzstreambase( name, mode), std::ostream( &buf) {}  
     gzstreambuf* rdbuf() { return gzstreambase::rdbuf(); }
-    void open( const filesystem::path & name, int open_mode = std::ios::out) {
+    void open( const std::filesystem::path & name, int open_mode = std::ios::out) {
         gzstreambase::open( name, open_mode);
     }
 };

libsrc/general/hashtabl.cpp

@@ -292,7 +292,19 @@ namespace netgen
 
 
 
-
+  BASE_INDEX_3_CLOSED_HASHTABLE ::
+  BASE_INDEX_3_CLOSED_HASHTABLE (size_t size)
+    : hash(RoundUp2(size))
+  {
+    // cout << "orig size = " << size
+    // << ", roundup size = " << hash.Size();
+    size = hash.Size();
+    mask = size-1;
+    // cout << "mask = " << mask << endl;
+    invalid = -1;
+    for (size_t i = 0; i < size; i++)
+      hash[i].I1() = invalid;
+  }
 
 
   void BASE_INDEX_3_CLOSED_HASHTABLE ::

libsrc/general/hashtabl.hpp

@@ -414,9 +414,14 @@ public:
     int BagNr() const { return bagnr; }
     int Pos() const { return pos; }
 
-    void operator++ (int)
+    Iterator operator++ (int)
+    {
+      Iterator it(ht, bagnr, pos);
+      ++(*this);
+      return it;
+    }
+    Iterator& operator++()
     {
-      // cout << "begin Operator ++: bagnr = " << bagnr << " -  pos = " << pos << endl;
       pos++;
       while (bagnr < ht.GetNBags() && 
 	     pos == ht.GetBagSize(bagnr+1))
@@ -424,7 +429,12 @@ public:
 	  pos = 0;
 	  bagnr++;
 	}
-      // cout << "end Operator ++: bagnr = " << bagnr << " - pos = " << pos << endl;
+      return *this;
+    }
+
+    std::pair<INDEX_3, T> operator*()
+    {
+      return std::make_pair(ht.hash[bagnr][pos], ht.cont[bagnr][pos]);
     }
 
     bool operator != (int i) const
@@ -446,6 +456,18 @@ public:
     return GetNBags();
   }
 
+  Iterator begin () const
+  {
+    Iterator it(*this, 0, -1);
+    it++;
+    return it;
+  }
+
+  int end() const
+  {
+    return GetNBags();
+  }
+
   void GetData (const Iterator & it,
 		INDEX_3 & ahash, T & acont) const
   {
@@ -839,9 +861,10 @@ inline ostream & operator<< (ostream & ost, const INDEX_2_CLOSED_HASHTABLE<T> &
   for (int i = 0; i < ht.Size(); i++)
     if (ht.UsedPos(i))
       {
-	INDEX_2 hash;
+	// INDEX_2 hash;
-	T data;
+	// T data;
-	ht.GetData0 (i, hash, data);
+	// ht.GetData0 (i, hash, data);
+        auto [hash,data] = ht.GetBoth(i);
 	ost << "hash = " << hash << ", data = " << data << endl;
       }
   return ost;
@@ -858,7 +881,8 @@ protected:
   size_t mask;
 
 protected: 
-  BASE_INDEX_3_CLOSED_HASHTABLE (size_t size)
+  BASE_INDEX_3_CLOSED_HASHTABLE (size_t size);
+  /*
     : hash(RoundUp2(size))
   {
     // cout << "orig size = " << size
@@ -870,6 +894,7 @@ protected:
     for (size_t i = 0; i < size; i++)
       hash[i].I1() = invalid;
   }
+  */
 
 public:
   int Size() const 
@@ -1051,9 +1076,12 @@ inline ostream & operator<< (ostream & ost, const INDEX_3_CLOSED_HASHTABLE<T> &
   for (int i = 0; i < ht.Size(); i++)
     if (ht.UsedPos(i))
       {
+        /*
 	INDEX_3 hash;
 	T data;
         ht.GetData (i, hash, data);
+        */
+        auto [hash, data] = ht.GetBoth();
 	ost << "hash = " << hash << ", data = " << data << endl;
       }
   return ost;

libsrc/general/mpi_interface.cpp

@@ -1,54 +0,0 @@
-/**************************************************************************/
-/* File:   mpi_interface.cpp                                              */
-/* Author: Joachim Schoeberl                                              */
-/* Date:   04. Apr. 97                                                    */
-/**************************************************************************/
-
-#ifdef OLD
-#include <mystdlib.h>
-#include <myadt.hpp>
-
-
-namespace netgen
-{
-
-
-#ifdef PARALLEL
-  
-  void MyMPI_SendCmd (const char * cmd)
-  {
-    int ntasks;
-    MPI_Comm_size(MPI_COMM_WORLD, &ntasks);
-
-    if(ntasks==1)
-      return;
-    
-    for (int dest = 1; dest < ntasks; dest++)
-      MPI_Send( (void*)cmd, (strlen(cmd)+1), MPI_CHAR, dest, MPI_TAG_CMD, MPI_COMM_WORLD);
-  }
-
-  string MyMPI_RecvCmd ()
-  {
-    MPI_Status status;
-    int flag;
-    int size_of_msg = -1;
-
-    MPI_Probe(0, MPI_TAG_CMD, MPI_COMM_WORLD, &status);
-    MPI_Get_count(&status, MPI_CHAR, &size_of_msg);
-
-    //char* buf = (char*)malloc(size_of_msg*sizeof(char));
-    char buf[100000]; //1MB should be enough...
-
-    MPI_Recv( &buf, size_of_msg, MPI_CHAR, 0, MPI_TAG_CMD, MPI_COMM_WORLD, &status);
-    
-    return string(buf);
-  }
-
-  // #else
-  // MPI_Comm MPI_COMM_WORLD, MPI_COMM_NULL;
-#endif
-}
-
-
-
-#endif

libsrc/general/mpi_interface.hpp

@@ -1,336 +0,0 @@
-braucht keiner mehr
-
-
-#ifdef XXXXXX
-
-#ifndef FILE_PARALLEL
-#define FILE_PARALLEL
-
-
-#ifdef VTRACE
-#include "vt_user.h"
-#else
-  #define VT_USER_START(n)
-  #define VT_USER_END(n)
-  #define VT_TRACER(n)
-#endif
-
-
-namespace netgen
-{
-
-#ifdef OLD
-#ifdef PARALLEL
-  template <class T>
-  inline MPI_Datatype MyGetMPIType ( ) 
-  { cerr << "ERROR in GetMPIType() -- no type found" << endl;return 0; }
-  template <>
-  inline MPI_Datatype MyGetMPIType<int> ( )
-  { return MPI_INT; }
-  template <>
-  inline MPI_Datatype MyGetMPIType<double> ( ) 
-  { return MPI_DOUBLE; }
-  template <>
-  inline MPI_Datatype MyGetMPIType<char> ( ) 
-  { return MPI_CHAR; }
-  template<>
-  inline MPI_Datatype MyGetMPIType<size_t> ( ) 
-  { return MPI_UINT64_T; }
-#else
-  typedef int MPI_Datatype;
-  template <class T> inline MPI_Datatype MyGetMPIType ( ) { return 0; }
-#endif
-#endif
-
-  
-  enum { MPI_TAG_CMD = 110 };
-  enum { MPI_TAG_MESH = 210 };
-  enum { MPI_TAG_VIS = 310 };
-
-#ifdef PARALLEL
-
-  [[deprecated("mympi_send int, use comm.Send instead")]]            
-  inline void MyMPI_Send (int i, int dest, int tag, MPI_Comm comm)
-  {
-    int hi = i;
-    MPI_Send( &hi, 1, MPI_INT, dest, tag, comm);
-  }
-  
-  [[deprecated("mympi_revc int, use comm.Recv instead")]]            
-  inline void MyMPI_Recv (int & i, int src, int tag, MPI_Comm comm)
-  {
-    MPI_Status status;
-    MPI_Recv( &i, 1, MPI_INT, src, tag, comm, &status);
-  }
-
-  [[deprecated("mympi_send string, use comm.Send instead")]]              
-  inline void MyMPI_Send (const string & s, int dest, int tag, MPI_Comm comm)
-  {
-    MPI_Send( const_cast<char*> (s.c_str()), s.length(), MPI_CHAR, dest, tag, comm);
-  }
-
-  [[deprecated("mympi_revc string, use comm.Recv instead")]]              
-  inline void MyMPI_Recv (string & s, int src, int tag, MPI_Comm comm)
-  {
-    MPI_Status status;
-    int len;
-    MPI_Probe (src, tag, MPI_COMM_WORLD, &status);
-    MPI_Get_count (&status, MPI_CHAR, &len);
-    s.assign (len, ' ');
-    MPI_Recv( &s[0], len, MPI_CHAR, src, tag, comm, &status);
-  }
-
- 
-
-  template <class T, int BASE>
-  [[deprecated("mympi_send ngflatarray, use comm.send instead")]]              
-  inline void MyMPI_Send (NgFlatArray<T, BASE> s, int dest, int tag, MPI_Comm comm)
-  {
-    MPI_Send( &s.First(), s.Size(), GetMPIType<T>(), dest, tag, comm);
-  }
-
-  template <class T, int BASE>
-  [[deprecated("mympi_recv ngflatarray, use comm.Recv instead")]]                
-  inline void MyMPI_Recv ( NgFlatArray<T, BASE> s, int src, int tag, MPI_Comm comm)
-  {
-    MPI_Status status;
-    MPI_Recv( &s.First(), s.Size(), GetMPIType<T>(), src, tag, comm, &status);
-  }
-
-  template <class T, int BASE>
-  [[deprecated("use ngcore - Array instead")]]
-  inline void MyMPI_Recv ( NgArray <T, BASE> & s, int src, int tag, MPI_Comm comm)
-  {
-    MPI_Status status;
-    int len;
-    MPI_Probe (src, tag, comm, &status);
-    MPI_Get_count (&status, GetMPIType<T>(), &len);
-
-    s.SetSize (len);
-    MPI_Recv( &s.First(), len, GetMPIType<T>(), src, tag, comm, &status);
-  }
-
-  template <class T, int BASE>
-  [[deprecated("use ngcore - Array instead")]]
-  inline int MyMPI_Recv ( NgArray <T, BASE> & s, int tag, MPI_Comm comm)
-  {
-    MPI_Status status;
-    int len;
-    MPI_Probe (MPI_ANY_SOURCE, tag, comm, &status);
-
-    int src = status.MPI_SOURCE;
-
-    MPI_Get_count (&status, GetMPIType<T>(), &len);
-
-    s.SetSize (len);
-    MPI_Recv( &s.First(), len, GetMPIType<T>(), src, tag, comm, &status);
-
-    return src;
-  }
-
-
-  /*
-  template <class T, int BASE>
-  inline void MyMPI_ISend (NgFlatArray<T, BASE> s, int dest, int tag, MPI_Request & request)
-  {
-    MPI_Isend( &s.First(), s.Size(), MyGetMPIType<T>(), dest, tag, MPI_COMM_WORLD, & request);
-  }
-
-
-  template <class T, int BASE>
-  inline void MyMPI_IRecv (NgFlatArray<T, BASE> s, int dest, int tag, MPI_Request & request)
-  {
-    MPI_Irecv( &s.First(), s.Size(), MyGetMPIType<T>(), dest, tag, MPI_COMM_WORLD, & request);
-  }
-  */
-
-  template <class T, int BASE>
-  [[deprecated("mympi_isend ngflatarray, use comm.send instead")]]
-  [[deprecated("use ngcore - Array instead")]]
-  inline MPI_Request MyMPI_ISend (NgFlatArray<T, BASE> s, int dest, int tag, MPI_Comm comm)
-  {
-    MPI_Request request;
-    MPI_Isend( &s.First(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
-    return request;
-  }
-
-  template <class T, int BASE>
-  [[deprecated("mympi_irecv ngflatarray, use comm.recv instead")]]                
-  inline MPI_Request MyMPI_IRecv (NgFlatArray<T, BASE> s, int dest, int tag, MPI_Comm comm)
-  {
-    MPI_Request request;
-    MPI_Irecv( &s.First(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
-    return request;
-  }
-  
-  /*
-  template <class T, int BASE>
-  inline void MyMPI_ISend (NgFlatArray<T, BASE> s, int dest, int tag)
-  {
-    MPI_Request request;
-    MPI_Isend( &s.First(), s.Size(), MyGetMPIType<T>(), dest, tag, MPI_COMM_WORLD, &request);
-    MPI_Request_free (&request);
-  }
-
-
-  template <class T, int BASE>
-  inline void MyMPI_IRecv (NgFlatArray<T, BASE> s, int dest, int tag)
-  {
-    MPI_Request request;
-    MPI_Irecv( &s.First(), s.Size(), MyGetMPIType<T>(), dest, tag, MPI_COMM_WORLD, &request);
-    MPI_Request_free (&request);
-  }
-  */
-
-
-
-  /*
-    send a table entry to each of the processes in the group ...
-    receive-table entries will be set
-   */
-
-  template <typename T>
-  [[deprecated("do we need that ? ")]]       
-  inline void MyMPI_ExchangeTable (TABLE<T> & send_data, 
-				   TABLE<T> & recv_data, int tag,
-				   const NgMPI_Comm & comm)
-  {
-    int rank = comm.Rank();
-    int ntasks = comm.Size();
-    
-    Array<int> send_sizes(ntasks);
-    Array<int> recv_sizes(ntasks);
-    for (int i = 0; i < ntasks; i++)
-      send_sizes[i] = send_data[i].Size();
-
-    comm.AllToAll (send_sizes, recv_sizes);
-    
-    for (int i = 0; i < ntasks; i++)
-      recv_data.SetEntrySize (i, recv_sizes[i], sizeof(T));
-    
-    Array<MPI_Request> requests;
-    for (int dest = 0; dest < ntasks; dest++)
-      if (dest != rank && send_data[dest].Size())
-        requests.Append (comm.ISend (FlatArray<T>(send_data[dest]), dest, tag));
-
-    for (int dest = 0; dest < ntasks; dest++)
-      if (dest != rank && recv_data[dest].Size())
-        requests.Append (comm.IRecv (FlatArray<T>(recv_data[dest]), dest, tag));
-
-    MyMPI_WaitAll (requests);
-  }
-
-
-  template <typename T>
-  [[deprecated("do we need that ? ")]]       
-  inline void MyMPI_ExchangeTable (DynamicTable<T> & send_data, 
-				   DynamicTable<T> & recv_data, int tag,
-				   const NgMPI_Comm & comm)
-  {
-    int rank = comm.Rank();
-    int ntasks = comm.Size();
-    
-    Array<int> send_sizes(ntasks);
-    Array<int> recv_sizes(ntasks);
-    for (int i = 0; i < ntasks; i++)
-      send_sizes[i] = send_data[i].Size();
-
-    comm.AllToAll (send_sizes, recv_sizes);
-    
-    // for (int i = 0; i < ntasks; i++)
-    // recv_data.SetEntrySize (i, recv_sizes[i], sizeof(T));
-    recv_data = DynamicTable<T> (recv_sizes, true);
-    
-    Array<MPI_Request> requests;
-    for (int dest = 0; dest < ntasks; dest++)
-      if (dest != rank && send_data[dest].Size())
-        requests.Append (comm.ISend (FlatArray<T>(send_data[dest]), dest, tag));
-
-    for (int dest = 0; dest < ntasks; dest++)
-      if (dest != rank && recv_data[dest].Size())
-        requests.Append (comm.IRecv (FlatArray<T>(recv_data[dest]), dest, tag));
-
-    MyMPI_WaitAll (requests);
-  }
-
-
-
-  
-  [[deprecated("do we still send commands?")]]                      
-  DLL_HEADER void MyMPI_SendCmd (const char * cmd);
-  [[deprecated("do we still send commands?")]]                        
-  extern string MyMPI_RecvCmd ();
-
-
-  template <class T>
-  [[deprecated("use comm.BCast instead")]]                      
-  inline void MyMPI_Bcast (T & s, MPI_Comm comm)
-  {
-    MPI_Bcast (&s, 1, GetMPIType<T>(), 0, comm);
-  }
-
-  template <class T>
-  [[deprecated("use comm.BCast instead")]]                        
-  inline void MyMPI_Bcast (NgArray<T, 0> & s, NgMPI_Comm comm)
-  {
-    int size = s.Size();
-    // MyMPI_Bcast (size, comm);
-    comm.Bcast(size);
-    // if (MyMPI_GetId(comm) != 0) s.SetSize (size);
-    if (comm.Rank() != 0) s.SetSize (size);
-    MPI_Bcast (&s[0], size, GetMPIType<T>(), 0, comm);
-  }
-
-  template <class T>
-  [[deprecated("use comm.BCast instead")]]                        
-  inline void MyMPI_Bcast (NgArray<T, 0> & s, int root, MPI_Comm comm)
-  {
-    int id;
-    MPI_Comm_rank(comm, &id);
-
-    int size = s.Size();
-    MPI_Bcast (&size, 1, MPI_INT, root, comm);
-    if (id != root) s.SetSize (size);
-    if ( !size ) return;
-    MPI_Bcast (&s[0], size, GetMPIType<T>(), root, comm);
-  }
-
-  template <class T, class T2>
-  [[deprecated("mympi_allgather deprecated, use comm.allgather")]]                    
-  inline void MyMPI_Allgather (const T & send, NgFlatArray<T2> recv, MPI_Comm comm)
-  {
-    MPI_Allgather( const_cast<T*> (&send), 1, GetMPIType<T>(), &recv[0], 1, GetMPIType<T2>(), comm);
-  }
-
-  template <class T, class T2>
-  [[deprecated("mympi_alltoall deprecated, use comm.alltoall")]]                  
-  inline void MyMPI_Alltoall (NgFlatArray<T> send, NgFlatArray<T2> recv, MPI_Comm comm)
-  {
-    MPI_Alltoall( &send[0], 1, GetMPIType<T>(), &recv[0], 1, GetMPIType<T2>(), comm);
-  }
-
-
-#else
-  template <typename T>
-  [[deprecated("do we need that ? ")]]       
-  inline void MyMPI_ExchangeTable (TABLE<T> & send_data, 
-				   TABLE<T> & recv_data, int tag,
-				   const NgMPI_Comm & comm)
-  {
-    ;
-  }
-
-  template <typename T>
-  [[deprecated("do we need that ? ")]]       
-  inline void MyMPI_ExchangeTable (DynamicTable<T> & send_data, 
-				   DynamicTable<T> & recv_data, int tag,
-				   const NgMPI_Comm & comm)
-  { ; } 
-#endif // PARALLEL
-
-}
-
-#endif
-
-
-#endif

libsrc/general/myadt.hpp

@@ -45,5 +45,4 @@ namespace netgen
 // #include "mpi_interface.hpp"
 #include "netgenout.hpp"
 
-
 #endif

libsrc/general/mystring.cpp

@@ -223,6 +223,16 @@ MyStr::MyStr(const string & st)
   strcpy (str, st.c_str());
 }
 
+MyStr::MyStr(string_view sv)
+{
+  length = unsigned(sv.length());
+  if (length > SHORTLEN)
+    str = new char[length + 1];
+  else
+    str = shortstr;
+  strcpy (str, sv.data());
+}
+
 MyStr::MyStr(const filesystem::path & path)
     : MyStr(path.string())
 { }

libsrc/general/mystring.hpp

@@ -60,6 +60,7 @@ public:
   MyStr(const Point3d& p);
   MyStr(const Vec3d& p);
   MyStr(const string & st);
+  MyStr(string_view sv);
   MyStr(const filesystem::path & st);
 
   ~MyStr();

libsrc/general/ngarray.cpp

@@ -1,75 +0,0 @@
-#ifndef FILE_NGSTD_NgArrayCPP
-#define FILE_NGSTD_NgArrayCPP
-// necessary for SGI ????
-
-/**************************************************************************/
-/* File:   array.cpp                                                       */
-/* Author: Joachim Schoeberl                                              */
-/* Date:   01. Jun. 95                                                    */
-/**************************************************************************/
-
-/* 
-   Abstract data type NgArray
-*/
-
-#include <mystdlib.h>
-#include <myadt.hpp>
-#include <assert.h>
-
-
-namespace netgen
-{
-  //using namespace netgen;
-
-#ifdef NONE  
-  void BASE_Array :: ReSize (int minsize, int elementsize)
-  {
-    cout << "resize, minsize = " << minsize << endl;
-
-    if (inc == -1)
-      throw Exception ("Try to resize fixed size array");
-
-    
-    void * p;
-    int nsize = (inc) ? allocsize + inc : 2 * allocsize;
-    if (nsize < minsize) nsize = minsize;
-
-    if (data)
-      {
-	p = new char [nsize * elementsize];
-	
-	int mins = (nsize < actsize) ? nsize : actsize; 
-	memcpy (p, data, mins * elementsize);
-	
-	delete [] static_cast<char*> (data);
-	data = p;
-      }
-    else
-      {
-	data = new char[nsize * elementsize];
-      }
-    
-    allocsize = nsize;
-    cout << "resize done" << endl;
-  }
-  
-  
-  
-  void BASE_Array :: RangeCheck (int i) const
-  {
-    if (i < 0 || i >= actsize)
-      throw ArrayRangeException ();
-  }
-  
-  void BASE_Array :: CheckNonEmpty () const
-  {
-    if (!actsize)
-      {
-	throw Exception ("NgArray should not be empty");
-	//      cerr << "NgArray shouldn't be empty";
-      }
-  }
-#endif
-}
-#endif // FILE_NGSTD_NgArrayCPP
-

libsrc/general/ngarray.hpp

@@ -11,6 +11,7 @@
 
 namespace netgen
 {
+  using namespace ngcore;
 
   // template <class T, int B1, int B2> class IndirectArray;
   template <class TA1, class TA2> class NgIndirectArray;
@@ -111,11 +112,7 @@ namespace netgen
     /// Access array. BASE-based
     T & operator[] (TIND i) const
     {
-#ifdef DEBUG
+      NETGEN_CHECK_RANGE(i,BASE,size+BASE);
-      if (i-BASE < 0 || i-BASE >= size)
-	cout << "array<" << typeid(T).name() << "> out of range, i = " << i << ", s = " << size << endl;
-#endif
-
       return data[i-BASE]; 
     }
 
@@ -130,13 +127,7 @@ namespace netgen
     /// Access array, one-based  (old fashioned)
     T & Elem (int i)
     {
-#ifdef DEBUG
+      NETGEN_CHECK_RANGE(i,1,size+1);
-      if (i < 1 || i > size)
-	cout << "NgArray<" << typeid(T).name() 
-	     << ">::Elem out of range, i = " << i
-	     << ", s = " << size << endl;
-#endif
-
       return ((T*)data)[i-1]; 
     }
   
@@ -144,30 +135,21 @@ namespace netgen
     // [[deprecated("Use operator[] instead")]]    
     const T & Get (int i) const 
     {
-#ifdef DEBUG
+      NETGEN_CHECK_RANGE(i,1,size+1);
-      if (i < 1 || i > size)
-	cout << "NgArray<" << typeid(T).name() << ">::Get out of range, i = " << i
-	     << ", s = " << size << endl;
-#endif
-
       return ((const T*)data)[i-1]; 
     }
 
     /// Access array, one-based  (old fashioned)
     void Set (int i, const T & el)
     { 
-#ifdef DEBUG
+      NETGEN_CHECK_RANGE(i,1,size+1);
-      if (i < 1 || i > size)
-	cout << "NgArray<" << typeid(T).name() << ">::Set out of range, i = " << i
-	     << ", s = " << size << endl;
-#endif
-
       ((T*)data)[i-1] = el; 
     }
 
     /// access first element
     T & First () const
     {
+      NETGEN_CHECK_RANGE(0,0,size);
       return data[0];
     }
 
@@ -175,6 +157,7 @@ namespace netgen
     /// access last element. check by macro CHECK_RANGE
     T & Last () const
     {
+      NETGEN_CHECK_RANGE(size-1,0,size);
       return data[size-1];
     }
 
@@ -345,10 +328,7 @@ namespace netgen
     /// Delete element i (0-based). Move last element to position i.
     void Delete (TIND i)
     {
-#ifdef CHECK_Array_RANGE
+      NETGEN_CHECK_RANGE(i,0,size);
-      RangeCheck (i+1);
-#endif
-
       data[i] = std::move(data[size-1]);
       size--;
       //    DeleteElement (i+1);
@@ -358,10 +338,7 @@ namespace netgen
     /// Delete element i (1-based). Move last element to position i.
     void DeleteElement (TIND i)
     {
-#ifdef CHECK_Array_RANGE
+      NETGEN_CHECK_RANGE(i,1,size+1);
-      RangeCheck (i);
-#endif
-
       data[i-1] = std::move(data[size-1]);
       size--;
     }

libsrc/general/ngbitarray.hpp

@@ -12,7 +12,6 @@
 namespace netgen
 {
 
-
 /**
    data type NgBitArray
    
@@ -29,8 +28,11 @@ class NgBitArray
   unsigned char * data;
 
 public:
+  
+  // [[ deprecated ("use BitArray instead")]]  
   DLL_HEADER NgBitArray ();
   ///
+  // [[ deprecated ("use BitArray instead")]]  
   DLL_HEADER NgBitArray (INDEX asize);
   ///
   DLL_HEADER ~NgBitArray ();

libsrc/general/ngpython.hpp

@@ -8,7 +8,7 @@
 #include <iostream>
 #include <sstream>
 
-using namespace ngcore;
+// using namespace ngcore;
 
 template <typename T>
 py::array MoveToNumpy(std::vector<T>& vec)

libsrc/general/parthreads.hpp

@@ -77,8 +77,8 @@ public:
 template<typename TFunc>
 void ParallelFor( int first, int next, const TFunc & f )
 {
-  int nthreads = thread::hardware_concurrency();
+  int nthreads = std::thread::hardware_concurrency();
-  thread * threads = new thread[nthreads];
+  std::thread * threads = new std::thread[nthreads];
   for (int i=0; i<nthreads; i++)
     {
       int myfirst = first + (next-first)*i/nthreads;
@@ -97,7 +97,7 @@ void ParallelFor( int first, int next, const TFunc & f )
 
   
   typedef void (*NgTaskManager)(std::function<void(int,int)>);
-  typedef void (*NgTracer)(string, bool);  // false .. start, true .. stop
+  typedef void (*NgTracer)(std::string, bool);  // false .. start, true .. stop
 
   inline void DummyTaskManager (std::function<void(int,int)> func)
   {
@@ -105,7 +105,7 @@ void ParallelFor( int first, int next, const TFunc & f )
     func(1,2);
   }
 
-  inline void DummyTracer (string, bool) { ; }
+  inline void DummyTracer (std::string, bool) { ; }
   
   template <typename FUNC>
   inline void ParallelFor (NgTaskManager tm, size_t n, FUNC func)

libsrc/general/table.cpp

@@ -109,6 +109,7 @@ namespace netgen
       {
 	void * p = new char [(line.maxsize+5) * elsize];
 
+        if (line.maxsize && elsize)
           memcpy (p, line.col, line.maxsize * elsize);
 	delete [] (char*)line.col;
 

libsrc/general/table.hpp

@@ -120,7 +120,7 @@ public:
   
   /// Creates fixed maximal element size table
   inline TABLE (const NgFlatArray<int,BASE> & entrysizes)
-    : BASE_TABLE (NgFlatArray<int> (entrysizes.Size(), const_cast<int*>(&entrysizes[BASE])), 
+    : BASE_TABLE (NgFlatArray<int> (entrysizes.Size(), entrysizes.Size() ? const_cast<int*>(&entrysizes[BASE]) : nullptr), 
 		  sizeof(T))
   { ; }
 
@@ -169,6 +169,7 @@ public:
   /// Inserts element acont into row i. BASE-based. Does not test if already used, assumes to have enough memory
   inline void AddSave (int i, const T & acont)
     {
+      NETGEN_CHECK_RANGE(i, BASE, data.Size()+BASE);
       ((T*)data[i-BASE].col)[data[i-BASE].size] = acont;
       data[i-BASE].size++;
     }

libsrc/general/template.hpp

@@ -16,7 +16,7 @@ namespace netgen
    templates, global types, defines and variables
 */
 
-DLL_HEADER extern const string netgen_version;
+  DLL_HEADER extern const std::string netgen_version;
 
 ///	The following value may be adapted to the hardware !
 #ifndef CLOCKS_PER_SEC
@@ -114,14 +114,24 @@ class INDEX_2
 
 public:
   ///
+  // protected:
   INDEX_2 () { }
+  INDEX_2 (const INDEX_2&) = default;
+public:
+  INDEX_2 (INDEX_2&&) = default;
+
+  INDEX_2 & operator= (const INDEX_2&) = default;
+  INDEX_2 & operator= (INDEX_2&&) = default;
   ///
-  INDEX_2 (INDEX ai1, INDEX ai2)
+  constexpr INDEX_2 (INDEX ai1, INDEX ai2)
-    { i[0] = ai1; i[1] = ai2; }
+    : i{ai1, ai2} { } 
+  // { i[0] = ai1; i[1] = ai2; }
 
   ///
-  INDEX_2 (const INDEX_2 & in2)
+  // constexpr INDEX_2 (const INDEX_2 & in2)
-    { i[0] = in2.i[0]; i[1] = in2.i[1]; }
+  // : i{in2.i[0], in2.i[1]} { } 
+  
+  // { i[0] = in2.i[0]; i[1] = in2.i[1]; }
 
   ///
   int operator== (const INDEX_2 & in2) const
@@ -130,7 +140,7 @@ public:
   ///
 
 
-  INDEX_2 Sort ()
+  constexpr INDEX_2 Sort ()
   {
     if (i[0] > i[1]) 
       {
@@ -149,7 +159,7 @@ public:
       return INDEX_2 (i1,i2);
   }
 
-
+  operator std::array<INDEX,2>() { return { i[0], i[1] }; }
   ///
   INDEX & I1 () { return i[0]; }
   ///
@@ -165,7 +175,7 @@ public:
   ///
   int & operator[] (int j) { return i[j]; }
   ///
-  const int & operator[] (int j) const { return i[j]; }
+  constexpr const int & operator[] (int j) const { return i[j]; }
   ///
   friend ostream & operator<<(ostream  & s, const INDEX_2 & i2);
 };
@@ -203,13 +213,12 @@ public:
   ///
   INDEX_3 () { }
   ///
-  INDEX_3 (INDEX ai1, INDEX ai2, INDEX ai3)
+  constexpr INDEX_3 (INDEX ai1, INDEX ai2, INDEX ai3)
-    { i[0] = ai1; i[1] = ai2; i[2] = ai3; }
+    : i{ai1, ai2, ai3} { }
 
   /// 
-  INDEX_3 (const INDEX_3 & in2)
+  constexpr INDEX_3 (const INDEX_3 & in2)
-    { i[0] = in2.i[0]; i[1] = in2.i[1]; i[2] = in2.i[2]; }
+    : i{in2.i[0], in2.i[1], in2.i[2]} { } 
-
 
   static INDEX_3 Sort (INDEX_3 i3)
   {
@@ -464,4 +473,37 @@ void MergeSort (int size, T * data, T * help);
 
 }
 
+namespace ngcore
+{
+  // template <>
+  // constexpr inline netgen::INDEX_2 InvalidHash<netgen::INDEX_2> () { return netgen::INDEX_2{-1,-1}; }
+
+
+  template <>
+  struct CHT_trait<netgen::INDEX_2>
+  {
+    constexpr static inline netgen::INDEX_2 Invalid() { return { -1, -1 } ; }
+    constexpr static inline size_t HashValue (const netgen::INDEX_2 & hash, size_t mask)
+    { return HashValue2(IVec<2,netgen::INDEX>(hash[0], hash[1]), mask); }
+  };
+
+  
+}
+
+namespace netgen
+{
+  /*
+  inline size_t HashValue2 (const netgen::INDEX_2 & ind, size_t mask)
+  {
+    return HashValue2(IVec<2,netgen::INDEX>(ind[0], ind[1]), mask);
+  }
+  */
+  
+  inline size_t HashValue2 (const netgen::INDEX_3 & ind, size_t mask)
+  {
+    return HashValue2(IVec<3,netgen::INDEX>(ind[0], ind[1], ind[2]), mask);
+  }
+
+
+}
 #endif

libsrc/geom2d/csg2d.cpp

@@ -12,8 +12,6 @@
 
 namespace netgen
 {
-using ngcore::INT;
-
 constexpr static double EPSILON=0.000000001;
 
 void ComputeWeight( Spline & s, Point<2> p )
@@ -2037,13 +2035,13 @@ shared_ptr<netgen::SplineGeometry2d> CSG2d :: GenerateSplineGeometry()
     }
 
   netgen::BoxTree <2> solid_tree(box);
-  Array<INT<2>> loop_list;
+  Array<IVec<2>> loop_list;
 
   for(auto i : Range(solids))
     for(auto li : Range(solids[i].polys))
     {
       solid_tree.Insert(solids[i].polys[li].GetBoundingBox(), loop_list.Size());
-      loop_list.Append(INT<2>(i, li));
+      loop_list.Append(IVec<2>(i, li));
     }
 
   for(auto i1 : Range(solids))

libsrc/geom2d/genmesh2d.cpp

@@ -139,7 +139,9 @@ namespace netgen
 	    mark = spline.GetPoint (edgelength);
 	  
 	    {
-	      PointIndex pi1 = -1, pi2 = -1;
+	      PointIndex pi1{PointIndex::INVALID};
+	      PointIndex pi2{PointIndex::INVALID};
+
 	  
 	      Point3d mark3(mark(0), mark(1), 0);
 	      Point3d oldmark3(oldmark(0), oldmark(1), 0);
@@ -157,12 +159,12 @@ namespace netgen
 		if ( mesh[PointIndex(locsearch[k])].GetLayer() == spline.layer)
 		  pi2 = locsearch[k];
 
-	      if (pi1 == -1)
+	      if (!pi1.IsValid())
 		{
 		  pi1 = mesh.AddPoint(oldmark3, spline.layer);
 		  searchtree.Insert (oldmark3, pi1);
 		}
-	      if (pi2 == -1)
+	      if (!pi2.IsValid())
 		{
 		  pi2 = mesh.AddPoint(mark3, spline.layer);
 		  searchtree.Insert (mark3, pi2);
@@ -278,7 +280,7 @@ namespace netgen
           mesh2d.AddLockedPoint(npi);
           Element0d el(npi, npi);
           el.name = point.name;
-          mesh2d.SetCD2Name(npi, point.name);
+          mesh2d.SetCD2Name(npi-IndexBASE<PointIndex>()+1, point.name);
           mesh2d.pointelements.Append (el);
           searchtree.Insert (newp, npi);          
         }
@@ -292,20 +294,20 @@ namespace netgen
             Point<2> hnewp = (j == 1) ? splines[i]->StartPI() : splines[i]->EndPI();
             Point<3> newp(hnewp(0), hnewp(1), 0);
             int layer = GetSpline(i).layer;
-            int npi = -1;
+            PointIndex npi(PointIndex::INVALID);
-            for (PointIndex pi = PointIndex::BASE; 
+            for (PointIndex pi = IndexBASE<PointIndex>(); 
-                 pi < mesh2d.GetNP()+PointIndex::BASE; pi++)
+                 pi < mesh2d.GetNP()+IndexBASE<PointIndex>(); pi++)
               if (Dist2 (mesh2d.Point(pi), newp) < 1e-12 * diam2 && mesh2d.Point(pi).GetLayer() == layer)
                 npi = pi;
             
-	    if (npi == -1)
+	    if (!npi.IsValid())
 	      {
 		npi = mesh2d.AddPoint (newp, layer);
 		searchtree.Insert (newp, npi);
                 mesh2d.AddLockedPoint(npi);
-                Element0d el(npi, npi);
+                Element0d el(npi, npi-IndexBASE<PointIndex>()+1);
                 el.name = "";
-                mesh2d.SetCD2Name(npi, "");
+                mesh2d.SetCD2Name(npi-IndexBASE<PointIndex>()+1, "");
                 mesh2d.pointelements.Append (el);
 	      }
           }
@@ -463,8 +465,8 @@ namespace netgen
 	  PointIndex mpi(0);
 	  Point<2> gp = geometry.GetPoint(i);
 	  Point<3> gp3(gp(0), gp(1), 0);
-	  for (PointIndex pi = PointIndex::BASE; 
+	  for (PointIndex pi = IndexBASE<PointIndex>(); 
-	       pi < mesh->GetNP()+PointIndex::BASE; pi++)
+	       pi < mesh->GetNP()+IndexBASE<PointIndex>(); pi++)
 	    if (Dist2(gp3, (*mesh)[pi]) < mindist)
 	      {
 		mpi = pi;
@@ -521,8 +523,8 @@ namespace netgen
         { // tensor product mesh
           RegionTimer rt(t_tensor);
           
-          NgArray<PointIndex, PointIndex::BASE> nextpi(bnp);
+          Array<PointIndex, PointIndex> nextpi(bnp);
-          NgArray<int, PointIndex::BASE> si1(bnp), si2(bnp);
+          Array<int, PointIndex> si1(bnp), si2(bnp);
           // PointIndex firstpi;
           
           nextpi = -1;
@@ -551,7 +553,8 @@ namespace netgen
           PointIndex c1(0), c2, c3, c4;  // 4 corner points
           int nex = 1, ney = 1;
 
-          for (PointIndex pi = 1; pi <= si2.Size(); pi++)
+          // for (PointIndex pi = 1; pi <= si2.Size(); pi++)
+          for (PointIndex pi : si2.Range())
             if (si2[pi] != -1)
               { c1 = pi; break; }      
 
@@ -564,13 +567,17 @@ namespace netgen
           NgArray<PointIndex> pts ( (nex+1) * (ney+1) );   // x ... inner loop
           pts = -1;
 
-          for (PointIndex pi = c1, i = 0; pi != c2; pi = nextpi[pi], i++)
+          int i = 0;
+          for (PointIndex pi = c1; pi != c2; pi = nextpi[pi], i++)
             pts[i] = pi;
-          for (PointIndex pi = c2, i = 0; pi != c3; pi = nextpi[pi], i++)
+          i = 0;
+          for (PointIndex pi = c2; pi != c3; pi = nextpi[pi], i++)
             pts[(nex+1)*i+nex] = pi;
-          for (PointIndex pi = c3, i = 0; pi != c4; pi = nextpi[pi], i++)
+          i = 0;
+          for (PointIndex pi = c3; pi != c4; pi = nextpi[pi], i++)
             pts[(nex+1)*(ney+1)-i-1] = pi;
-          for (PointIndex pi = c4, i = 0; pi != c1; pi = nextpi[pi], i++)
+          i = 0;
+          for (PointIndex pi = c4; pi != c1; pi = nextpi[pi], i++)
             pts[(nex+1)*(ney-i)] = pi;
 
 

libsrc/geom2d/python_geom2d.cpp

@@ -401,10 +401,8 @@ NGCORE_API_EXPORT void ExportGeom2d(py::module &m)
 		{
                   MeshingParameters mp;
                   if(pars) mp = *pars;
-                  {
-                    py::gil_scoped_acquire aq;
                   CreateMPfromKwargs(mp, kwargs);
-                  }
+                  py::gil_scoped_release gil_release;
 		  auto mesh = make_shared<Mesh>();
                   mesh->SetGeometry(self);
                   SetGlobalMesh (mesh);
@@ -414,7 +412,6 @@ NGCORE_API_EXPORT void ExportGeom2d(py::module &m)
                     throw Exception("Meshing failed!");
 		  return mesh;
                 }, py::arg("mp") = nullopt,
-      py::call_guard<py::gil_scoped_release>(),
       meshingparameter_description.c_str())
     .def("_SetDomainTensorMeshing", &SplineGeometry2d::SetDomainTensorMeshing)
     ;
@@ -427,7 +424,8 @@ NGCORE_API_EXPORT void ExportGeom2d(py::module &m)
     .def(py::self-py::self)
     .def(py::self*py::self)
     .def(py::self+=py::self)
-    .def(py::self-=py::self)
+    // .def(py::self-=py::self) // false clange warning, see  https://github.com/pybind/pybind11/issues/1893
+    .def("__isub__", [](Solid2d& lhs, const Solid2d& rhs) { return lhs -= rhs; }, py::is_operator())
     .def(py::self*=py::self)
 
     .def("Mat", &Solid2d::Mat)
@@ -466,10 +464,8 @@ NGCORE_API_EXPORT void ExportGeom2d(py::module &m)
 		{
                   MeshingParameters mp;
                   if(pars) mp = *pars;
-                  {
-                    py::gil_scoped_acquire aq;
                     CreateMPfromKwargs(mp, kwargs);
-                  }
+                  py::gil_scoped_release gil_release;
 		  auto mesh = make_shared<Mesh>();
                   auto geo = self.GenerateSplineGeometry();
                   mesh->SetGeometry(geo);
@@ -480,7 +476,6 @@ NGCORE_API_EXPORT void ExportGeom2d(py::module &m)
                     throw Exception("Meshing failed!");
 		  return mesh;
                 }, py::arg("mp") = nullopt,
-      py::call_guard<py::gil_scoped_release>(),
       meshingparameter_description.c_str())
     ;
 

libsrc/gprim/adtree.hpp

@@ -970,7 +970,7 @@ public:
           Leaf *leaf1 = (Leaf*) ball_leaves.Alloc(); new (leaf1) Leaf();
           Leaf *leaf2 = (Leaf*) ball_leaves.Alloc(); new (leaf2) Leaf();
 
-          for (auto i : order.Range(isplit))
+          for (auto i : order.Range(0, isplit))
               leaf1->Add(leaf_index, leaf->p[i], leaf->index[i] );
           for (auto i : order.Range(isplit, N))
               leaf2->Add(leaf_index, leaf->p[i], leaf->index[i] );
@@ -1334,7 +1334,7 @@ public:
             leaves.Append(leaf2);
             leaves[leaf1->nr] = leaf1;
 
-            for (auto i : order.Range(isplit))
+            for (auto i : order.Range(0,isplit))
                 leaf1->Add(leaves, leaf_index, leaf->p[i], leaf->index[i] );
             for (auto i : order.Range(isplit, N))
                 leaf2->Add(leaves, leaf_index, leaf->p[i], leaf->index[i] );

libsrc/gprim/geom2d.cpp

@@ -128,7 +128,7 @@ Point2d CrossPoint (const Line2d & l1, const Line2d & l2)
 
 
 int CrossPointBarycentric (const Line2d & l1, const Line2d & l2,
-			   double & lam1, double & lam2)
+			   double & lam1, double & lam2, double eps)
 {
   // p = l1.1 + lam1 (l1.2-l1.1) = l2.1 + lam2 (l2.2-l2.1)
   double a11 = l1.p2.X() - l1.p1.X();
@@ -140,8 +140,11 @@ int CrossPointBarycentric (const Line2d & l1, const Line2d & l2,
   double b2 = l2.p1.Y() - l1.p1.Y();
   
   double det = a11*a22 - a12 * a21;
+  /*
   if (det == 0)
     return 1;
+  */
+  if (fabs (det) < eps * (fabs(a11*a22)+fabs(a12*a21))) return 1;
     
   lam1 = (a22 * b1 - a12 * b2) / det;
   lam2 = (a11 * b2 - a21 * b1) / det;

libsrc/gprim/geom2d.hpp

@@ -365,7 +365,7 @@ namespace netgen
     friend DLL_HEADER Point2d CrossPoint (const Line2d & l1, const Line2d & l2);
     /// returns 1 iff parallel
     friend int CrossPointBarycentric (const Line2d & l1, const Line2d & l2,
-				      double & lam1, double & lam2);
+				      double & lam1, double & lam2, double eps);
     
     ///
     friend int Parallel (const Line2d & l1, const Line2d & l2, double peps);

libsrc/gprim/geomobjects.hpp

@@ -140,7 +140,8 @@ namespace netgen
 
     operator const T* () const { return x; }
 
-    void DoArchive(Archive& archive)
+    template <typename ARCHIVE>
+    void DoArchive(ARCHIVE& archive)
     {
       for(int i=0; i<D; i++)
         archive & x[i];
@@ -294,7 +295,8 @@ namespace netgen
       sol = inv * rhs;
     }
 
-    void DoArchive(Archive & ar)
+    template <typename ARCHIVE>
+    void DoArchive(ARCHIVE & ar)
     {
       ar.Do(x, H*W);
     }
@@ -341,8 +343,6 @@ namespace netgen
           pmin(i) = 1e99;
           pmax(i) = -1e99;
         }
-      // pmin = Point<D> (1e99, 1e99, 1e99);
-      // pmax = Point<D> (-1e99, -1e99, -1e99);
     }
 
     const Point<D> & PMin () const { return pmin; }
@@ -442,7 +442,8 @@ namespace netgen
       pmax = center + factor*(pmax-center);
     }
 
-    void DoArchive(Archive& archive)
+    template <typename ARCHIVE>
+    void DoArchive(ARCHIVE & archive)
     { archive & pmin & pmax; }
   };
 

libsrc/include/incopengl.hpp

@@ -5,6 +5,11 @@
 #include <mystdlib.h>
 #include <mydefs.hpp>
 
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#endif
+
 #  ifdef __APPLE__
 #define GL_SILENCE_DEPRECATION
 #define GL_DO_NOT_WARN_IF_MULTI_GL_VERSION_HEADERS_INCLUDED

libsrc/include/inctcl.hpp

@@ -1,3 +1,8 @@
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#endif
+
 #include <tcl.h>
 #include <tk.h>
 

libsrc/include/mystdlib.h

@@ -20,57 +20,40 @@
 #include <mutex>
 #include <atomic>
 #include <optional>
+#include <cassert>
 
 #include <new>
 #include <string>
 #include <typeinfo>
 
-#ifdef PARALLEL
-// #undef SEEK_SET
-// #undef SEEK_CUR
-// #undef SEEK_END
-#include <mpi.h>
-#include <unistd.h>  // for usleep (only for parallel)
-#endif
-
-
-
-/*
-#ifdef METIS
-namespace metis { extern "C" {
-#include <metis.h>
-} }
-#endif
-*/
-
-
 
 #ifndef M_PI
 #define M_PI 3.14159265358979323846
 #endif
 
-
 /*** Windows headers ***/
 #ifdef _MSC_VER
 # define WIN32_LEAN_AND_MEAN
 # ifndef NO_PARALLEL_THREADS
 #  ifdef MSVC_EXPRESS
 #  else
-#   include <afxwin.h>
+// #   include <afxwin.h>
-#   include <afxmt.h>
+// #   include <afxmt.h>
 #  endif // MSVC_EXPRESS
 # endif
-# include <windows.h>
+// # include <windows.h>
 # undef WIN32_LEAN_AND_MEAN
-# include <winnt.h>
+// # include <winnt.h>
-
 #else // Not using MC VC++
+#endif
+
+
+// using namespace std;
+namespace netgen
+{
+  using namespace std;  
+}
 
 
 #endif
 
-
-using namespace std;
-
-#endif
-

libsrc/include/nginterface_v2.hpp

@@ -9,6 +9,7 @@
 /**************************************************************************/
 
 #include "mydefs.hpp"
+#include <core/mpi_wrapper.hpp>
 
 /*
   C++ interface to Netgen
@@ -38,24 +39,9 @@ namespace netgen
   using namespace std;
   using namespace ngcore;
   
-  // extern DLL_HEADER NgMPI_Comm ng_comm;
   
   static constexpr int POINTINDEX_BASE = 1;
 
-  /*
-  struct T_EDGE2
-  {
-    // int orient:1;
-    // int nr:31;    // 0-based
-    int nr;    // 0-based
-  };
-  struct T_FACE2
-  {
-    // int orient:3;
-    // int nr:29;    // 0-based
-    int nr;    // 0-based
-  };
-  */
   typedef int T_EDGE2; 
   typedef int T_FACE2; 
 
@@ -111,26 +97,6 @@ namespace netgen
       int operator[] (size_t i) const { return ptr[i]-POINTINDEX_BASE; }
     };
 
-    class Ng_Edges
-    {
-    public:
-      size_t num;
-      const T_EDGE2 * ptr;
-  
-      size_t Size() const { return num; }
-      int operator[] (size_t i) const { return ptr[i]; }
-    };
-
-    class Ng_Faces
-    {
-    public:
-      size_t num;
-      const T_FACE2 * ptr;
-  
-      size_t Size() const { return num; }
-      int operator[] (size_t i) const { return ptr[i]; }
-    };
-
     class Ng_Facets
     {
     public:
@@ -151,10 +117,11 @@ namespace netgen
     int GetIndex() const { return index-1; }
     Ng_Points points;      // all points
     Ng_Vertices vertices;
-    Ng_Edges edges;
+    FlatArray<T_EDGE2> edges;
-    Ng_Faces faces;
+    FlatArray<T_FACE2> faces;    
     Ng_Facets facets;
     bool is_curved;
+    int8_t newest_vertex;
   };
 
   
@@ -407,6 +374,10 @@ namespace netgen
     int GetClusterRepEdge (int edi) const;
     int GetClusterRepFace (int fai) const;
     int GetClusterRepElement (int eli) const;
+
+    // just copied from nginterface, now 0-based
+    int GetElement_Faces (int elnr, int * faces, int * orient = 0) const;
+    int GetSurfaceElement_Face (int selnr, int * orient = 0) const;
   };
 
 

libsrc/include/nginterface_v2_impl.hpp

@@ -57,23 +57,32 @@ NGX_INLINE DLL_HEADER Ng_Element Ngx_Mesh :: GetElement<0> (size_t nr) const
   ret.vertices.num = 1;
   ret.vertices.ptr = (int*)&el.pnum;
 
+  /*
   ret.edges.num = 0;
   ret.edges.ptr = NULL;
-  
+  */
+  ret.edges.Assign ( FlatArray<T_EDGE2> (0, nullptr) );
+  /*
   ret.faces.num = 0;
   ret.faces.ptr = NULL;
+  */
+  ret.faces.Assign ( { 0, nullptr } );  
   
   ret.facets.num = 1;
-  ret.facets.base = 1;
+  ret.facets.base = POINTINDEX_BASE;
   ret.facets.ptr = (int*)&el.pnum;
 
+  /*
   if (mesh->GetDimension() == 1)
     ret.mat = *(mesh->GetBCNamePtr(el.index-1));
   else if (mesh->GetDimension() == 2)
     ret.mat = *(mesh->GetCD2NamePtr(el.index-1));
   else
     ret.mat = *(mesh->GetCD3NamePtr(el.index-1));
+  */
+  ret.mat = mesh->GetRegionName(0, el.index);
     
+  ret.is_curved = false;
   return ret;
 }
 
@@ -91,6 +100,8 @@ NGX_INLINE DLL_HEADER Ng_Element Ngx_Mesh :: GetElement<1> (size_t nr) const
     ret.index = el.edgenr;
   else
     ret.index = el.si;
+
+  /*
   if (mesh->GetDimension() == 2)
     ret.mat = *(mesh->GetBCNamePtr(el.si-1));
   else
@@ -100,6 +111,8 @@ NGX_INLINE DLL_HEADER Ng_Element Ngx_Mesh :: GetElement<1> (size_t nr) const
       else
         ret.mat = *(mesh->GetMaterialPtr(el.si));
     }
+  */
+  ret.mat = mesh->GetRegionName(1, ret.index);
 
   ret.points.num = el.GetNP();
   ret.points.ptr = (int*)&(el[0]);
@@ -107,11 +120,17 @@ NGX_INLINE DLL_HEADER Ng_Element Ngx_Mesh :: GetElement<1> (size_t nr) const
   ret.vertices.num = 2;
   ret.vertices.ptr = (int*)&(el[0]);
 
+  /*
   ret.edges.num = 1;
   ret.edges.ptr = mesh->GetTopology().GetSegmentElementEdgesPtr (nr);
+  */
+  ret.edges.Assign ( FlatArray<T_EDGE2> (1, const_cast<T_EDGE2*>((const int*)  mesh->GetTopology().GetSegmentElementEdgesPtr (nr))));
 
+  /*
   ret.faces.num = 0;
   ret.faces.ptr = NULL;
+  */
+  ret.faces.Assign ( { 0, nullptr });
   
   if (mesh->GetDimension() == 3)
     {
@@ -123,12 +142,12 @@ NGX_INLINE DLL_HEADER Ng_Element Ngx_Mesh :: GetElement<1> (size_t nr) const
     {
       ret.facets.num = 1;
       ret.facets.base = 0;
-      ret.facets.ptr = ret.edges.ptr;
+      ret.facets.ptr = ret.edges.Data();
     }
   else
     {
       ret.facets.num = 2;
-      ret.facets.base = 1;
+      ret.facets.base = POINTINDEX_BASE;
       ret.facets.ptr = (int*)&(el[0]);
     }
 
@@ -157,25 +176,37 @@ NGX_INLINE DLL_HEADER Ng_Element Ngx_Mesh :: GetElement<2> (size_t nr) const
   ret.vertices.num = el.GetNV();
   ret.vertices.ptr = (int*)&(el[0]);
 
+  /*
   ret.edges.num = MeshTopology::GetNEdges (el.GetType());
   ret.edges.ptr = mesh->GetTopology().GetSurfaceElementEdgesPtr (nr);
+  */
 
+  // ret.edges.Assign (mesh->GetTopology().GetEdges (SurfaceElementIndex(nr)));
+  auto hedges = mesh->GetTopology().GetEdges (SurfaceElementIndex(nr));
+  ret.edges.Assign ( { hedges.Size(), (int*)hedges.Data() } );
+  
+  /*
   ret.faces.num = MeshTopology::GetNFaces (el.GetType());
   ret.faces.ptr = mesh->GetTopology().GetSurfaceElementFacesPtr (nr);
+  */
+
+  // ret.faces.Assign ( { 1, const_cast<int*>(mesh->GetTopology().GetSurfaceElementFacesPtr (nr)) });
+  ret.faces.Assign ( { 1, (int*)(mesh->GetTopology().GetSurfaceElementFacesPtr (nr)) });
   
   if (mesh->GetDimension() == 3)
     {
-      ret.facets.num = ret.faces.num;
+      ret.facets.num = ret.faces.Size();
       ret.facets.base = 0;
-      ret.facets.ptr = ret.faces.ptr;
+      ret.facets.ptr = ret.faces.Data();
     }
   else
     {
-      ret.facets.num = ret.edges.num;
+      ret.facets.num = ret.edges.Size();
       ret.facets.base = 0;      
-      ret.facets.ptr = ret.edges.ptr;
+      ret.facets.ptr = ret.edges.Data();
     }
   ret.is_curved = el.IsCurved();
+  ret.newest_vertex = el.NewestVertex();
   return ret;
 }
 
@@ -194,17 +225,29 @@ NGX_INLINE DLL_HEADER Ng_Element Ngx_Mesh :: GetElement<3> (size_t nr) const
   ret.vertices.num = el.GetNV();
   ret.vertices.ptr = (int*)&(el[0]);
 
+  /*
   ret.edges.num = MeshTopology::GetNEdges (el.GetType());
   ret.edges.ptr = mesh->GetTopology().GetElementEdgesPtr (nr);
+  */
+  // ret.edges.Assign (mesh->GetTopology().GetEdges (ElementIndex(nr)));
+  auto hedges = mesh->GetTopology().GetEdges (ElementIndex(nr));
+  ret.edges.Assign ( { hedges.Size(), (int*)hedges.Data() } );
   
+
+  /*
   ret.faces.num = MeshTopology::GetNFaces (el.GetType());
   ret.faces.ptr = mesh->GetTopology().GetElementFacesPtr (nr);
+  */
+  // ret.faces.Assign (mesh->GetTopology().GetFaces (ElementIndex(nr)));
+  auto hfaces = mesh->GetTopology().GetFaces (ElementIndex(nr));
+  ret.faces.Assign ( { hfaces.Size(), (int*)hfaces.Data() } );
   
-  ret.facets.num = ret.faces.num;
+  ret.facets.num = ret.faces.Size();
   ret.facets.base = 0;
-  ret.facets.ptr = ret.faces.ptr;
+  ret.facets.ptr = ret.faces.Data();
 
   ret.is_curved = el.IsCurved();
+  ret.newest_vertex = el.NewestVertex();
   return ret;
 }
 
@@ -248,10 +291,10 @@ template <> NGX_INLINE DLL_HEADER int Ngx_Mesh :: GetNNodes<2> ()
   return mesh->GetTopology().GetNFaces();
 }
 
-template <> NGX_INLINE DLL_HEADER const Ng_Node<0> Ngx_Mesh :: GetNode<0> (int vnr) const
+template <> NGX_INLINE DLL_HEADER const Ng_Node<0> Ngx_Mesh :: GetNode<0> (int vnr_) const
 {
   Ng_Node<0> node;
-  vnr++;
+  PointIndex vnr = IndexBASE<PointIndex>() + vnr_;
   switch (mesh->GetDimension())
     {
     case 3:
@@ -304,8 +347,8 @@ template <> NGX_INLINE DLL_HEADER const Ng_Node<2> Ngx_Mesh :: GetNode<2> (int n
 {
   Ng_Node<2> node;
   node.vertices.ptr = (const int*)mesh->GetTopology().GetFaceVerticesPtr(nr);
-  node.vertices.nv = (node.vertices.ptr[3] == 0) ? 3 : 4;
+  node.vertices.nv = (node.vertices.ptr[3]+1 == PointIndex::BASE) ? 3 : 4;
-  node.surface_el = mesh->GetTopology().GetFace2SurfaceElement (nr+1)-1;
+  node.surface_el = mesh->GetTopology().GetFace2SurfaceElement (nr);
   return node;
 }
 
@@ -316,8 +359,8 @@ NGX_INLINE DLL_HEADER Ng_Buffer<int[2]> Ngx_Mesh :: GetPeriodicVertices(int idnr
   mesh->GetIdentifications().GetPairs (idnr+1, apairs);
   for(auto& ind : apairs)
     {
-      ind.I1()--;
+      ind.I1() -= IndexBASE<PointIndex>();
-      ind.I2()--;
+      ind.I2() -= IndexBASE<PointIndex>();
     }
   typedef int ti2[2];
   return { apairs.Size(), (ti2*)(void*)apairs.Release() };
@@ -326,12 +369,9 @@ NGX_INLINE DLL_HEADER Ng_Buffer<int[2]> Ngx_Mesh :: GetPeriodicVertices(int idnr
 
 NGX_INLINE void Ngx_Mesh :: GetParentNodes (int ni, int * parents) const
 {
-  ni++;
+  if (ni < mesh->mlbetweennodes.Size())
-  if (ni <= mesh->mlbetweennodes.Size())
+    for (int j = 0; j < 2; j++)
-    {
+      parents[j] = mesh->mlbetweennodes[IndexBASE<PointIndex>()+ni][j] - IndexBASE<PointIndex>();
-      parents[0] = mesh->mlbetweennodes.Get(ni).I1()-1;
-      parents[1] = mesh->mlbetweennodes.Get(ni).I2()-1;
-      }
   else
     parents[0] = parents[1] = -1;
 }

libsrc/interface/nginterface.cpp

@@ -27,17 +27,7 @@ namespace netgen
 static std::thread meshingthread;
 void RunParallel ( void * (*fun)(void *), void * in)
 {
-  bool parthread = netgen::mparam.parthread;
+  if (netgen::mparam.parthread)
-
-#ifdef PARALLEL
-  int provided;
-  MPI_Query_thread(&provided);
-  if (provided < 3)
-    if (netgen::ntasks > 1) parthread = false;
-  // cout << "runparallel = " << parthread << endl;
-#endif
-
-  if (parthread)
     {
       meshingthread = std::thread(fun, in);
       meshingthread.detach();
@@ -517,7 +507,7 @@ NG_ELEMENT_TYPE Ng_GetSurfaceElement (int ei, int * epi, int * np)
     {
       const Segment & seg = mesh->LineSegment (ei);
 
-      if (seg[2] < 0)
+      if (!seg[2].IsValid())
 	{
 	  epi[0] = seg[0];
 	  epi[1] = seg[1];
@@ -661,14 +651,14 @@ int Ng_FindElementOfPoint (double * p, double * lami, int build_searchtree,
     {
       Point3d p3d(p[0], p[1], p[2]);
       ind = 
-	mesh->GetElementOfPoint(p3d, lami, dummy, build_searchtree != 0);
+	mesh->GetElementOfPoint(p3d, lami, dummy, build_searchtree != 0) + 1;
     }
   else
     {
       double lam3[3];
       Point3d p2d(p[0], p[1], 0);
       ind = 
-	mesh->GetElementOfPoint(p2d, lam3, dummy, build_searchtree != 0);
+	mesh->GetSurfaceElementOfPoint(p2d, lam3, dummy, build_searchtree != 0) + 1;
 
       if (ind > 0)
 	{
@@ -707,7 +697,7 @@ int Ng_FindSurfaceElementOfPoint (double * p, double * lami, int build_searchtre
     {
       Point3d p3d(p[0], p[1], p[2]);
       ind = 
-	mesh->GetSurfaceElementOfPoint(p3d, lami, dummy, build_searchtree != 0);
+	mesh->GetSurfaceElementOfPoint(p3d, lami, dummy, build_searchtree != 0) + 1;
     }
   else
     {
@@ -876,7 +866,7 @@ NG_ELEMENT_TYPE Ng_GetSegment (int ei, int * epi, int * np)
   epi[0] = seg[0];
   epi[1] = seg[1];
 
-  if (seg[2] < 0)
+  if (!seg[2].IsValid())
     {
       if (np) *np = 2;
       return NG_SEGM;
@@ -1581,10 +1571,11 @@ int Ng_GetSurfaceElement_Face (int selnr, int * orient)
 {
   if (mesh->GetDimension() == 3)
     {
+      SurfaceElementIndex sei = selnr-1;
       const MeshTopology & topology = mesh->GetTopology();
       if (orient)
 	*orient = topology.GetSurfaceElementFaceOrientation (selnr);
-      return topology.GetSurfaceElementFace (selnr);
+      return topology.GetFace(sei);
     }
   return -1;
 }
@@ -1615,7 +1606,10 @@ void Ng_GetEdge_Vertices (int ednr, int * vert)
 {
   const MeshTopology & topology = mesh->GetTopology();
   // topology.GetEdgeVertices (ednr, vert[0], vert[1]);
-  tie(vert[0], vert[1]) = topology.GetEdgeVertices(ednr-1);
+  // tie(vert[0], vert[1]) = topology.GetEdgeVertices(ednr-1);
+  auto [v1,v2] = topology.GetEdgeVertices(ednr-1);
+  vert[0] = v1-IndexBASE<PointIndex>()+1;
+  vert[1] = v2-IndexBASE<PointIndex>()+1;
 }
 
 
@@ -1746,8 +1740,8 @@ void Ng_GetParentNodes (int ni, int * parents)
 {
   if (ni <= mesh->mlbetweennodes.Size())
     {
-      parents[0] = mesh->mlbetweennodes.Get(ni).I1();
+      parents[0] = mesh->mlbetweennodes[ni].I1();
-      parents[1] = mesh->mlbetweennodes.Get(ni).I2();
+      parents[1] = mesh->mlbetweennodes[ni].I2();
     }
   else
     parents[0] = parents[1] = 0;
@@ -1759,12 +1753,12 @@ int Ng_GetParentElement (int ei)
   if (mesh->GetDimension() == 3)
     {
       if (ei <= mesh->mlparentelement.Size())
-	return mesh->mlparentelement.Get(ei);
+	return mesh->mlparentelement[ei-1]+1;
     }
   else
     {
       if (ei <= mesh->mlparentsurfaceelement.Size())
-	return mesh->mlparentsurfaceelement.Get(ei);
+	return mesh->mlparentsurfaceelement[ei-1]+1;
     }
   return 0;
 }
@@ -1775,7 +1769,7 @@ int Ng_GetParentSElement (int ei)
   if (mesh->GetDimension() == 3)
     {
       if (ei <= mesh->mlparentsurfaceelement.Size())
-	return mesh->mlparentsurfaceelement.Get(ei);
+	return mesh->mlparentsurfaceelement[ei-1]+1;
     }
   else
     {
@@ -1837,7 +1831,7 @@ void Ng_GetPeriodicVertices (int idnr, int * pairs)
 
 int Ng_GetNPeriodicEdges (int idnr)
 {
-  NgArray<int,PointIndex::BASE> map;
+  idmap_type map;
   //const MeshTopology & top = mesh->GetTopology();
   int nse = mesh->GetNSeg();
 
@@ -1864,7 +1858,7 @@ int Ng_GetNPeriodicEdges (int idnr)
 
 void Ng_GetPeriodicEdges (int idnr, int * pairs)
 {
-  NgArray<int,PointIndex::BASE> map;
+  idmap_type map;
   const MeshTopology & top = mesh->GetTopology();
   int nse = mesh->GetNSeg();
 
@@ -1955,8 +1949,10 @@ int Ng_GetVertex_Elements( int vnr, int* elems )
 }
 
 ///// Added by Roman Stainko ....
-int Ng_GetVertex_SurfaceElements( int vnr, int* elems )
+int Ng_GetVertex_SurfaceElements( int vnr_, int* elems )
 {
+  PointIndex vnr = vnr_ + IndexBASE<PointIndex>()-1;
+  
   switch (mesh->GetDimension())
     {
     case 3:
@@ -2004,8 +2000,9 @@ int Ng_GetVertex_NElements( int vnr )
 }
 
 ///// Added by Roman Stainko ....
-int Ng_GetVertex_NSurfaceElements( int vnr )
+int Ng_GetVertex_NSurfaceElements( int vnr_ )
 {
+  PointIndex vnr = vnr_ + IndexBASE<PointIndex>()-1;
   switch (mesh->GetDimension())
     {
     case 3:
@@ -2250,17 +2247,14 @@ int Ng_GetClosureNodes (int nt, int nodenr, int nodeset, int * nodes)
             for (int i = 0; i < el.GetNP(); i++)
               { 
                 nodes[cnt++] = 0;
-                nodes[cnt++] = el[i] - PointIndex::BASE;
+                nodes[cnt++] = el[i] - IndexBASE<PointIndex>();
               }
           }
 
         if (nodeset & 2)  // Edges
           {
-            int edges[12];
+            auto edges = mesh->GetTopology().GetEdges (ElementIndex(nodenr));
-            // int ned;
+            for (int i = 0; i < edges.Size(); i++)
-            // ned = mesh->GetTopology().GetElementEdges (nodenr+1, edges, 0);
-            int ned = mesh->GetTopology().GetEdges (ElementIndex(nodenr)).Size();
-            for (int i = 0; i < ned; i++)
               {
                 nodes[cnt++] = 1;
                 nodes[cnt++] = edges[i]-1;
@@ -2335,7 +2329,7 @@ int Ng_GetElementClosureNodes (int dim, int elementnr, int nodeset, int * nodes)
             for (int i = 0; i < el.GetNP(); i++)
               { 
                 nodes[cnt++] = 0;
-                nodes[cnt++] = el[i] - PointIndex::BASE;
+                nodes[cnt++] = el[i] - IndexBASE<PointIndex>();
               }
           }
 
@@ -2353,7 +2347,7 @@ int Ng_GetElementClosureNodes (int dim, int elementnr, int nodeset, int * nodes)
 
         if (nodeset & 4)  // Faces
           {
-            int face = mesh->GetTopology().GetSurfaceElementFace (elementnr+1);
+            int face = mesh->GetTopology().GetFace (SurfaceElementIndex(elementnr))+1;
             nodes[cnt++] = 2;
             nodes[cnt++] = face-1;
           }

libsrc/interface/nginterface_v2.cpp

@@ -725,16 +725,15 @@ namespace netgen
   
   int Ngx_Mesh :: GetParentElement (int ei) const
   {
-      ei++;
     if (mesh->GetDimension() == 3)
       {
-          if (ei <= mesh->mlparentelement.Size())
+        if (ei < mesh->mlparentelement.Size())
-              return mesh->mlparentelement.Get(ei)-1;
+          return mesh->mlparentelement[ei];
       }
     else
       {
-          if (ei <= mesh->mlparentsurfaceelement.Size())
+        if (ei < mesh->mlparentsurfaceelement.Size())
-              return mesh->mlparentsurfaceelement.Get(ei)-1;
+          return mesh->mlparentsurfaceelement[ei];
       }
     return -1;
   }
@@ -742,11 +741,10 @@ namespace netgen
 
   int Ngx_Mesh :: GetParentSElement (int ei) const
   {
-      ei++;
     if (mesh->GetDimension() == 3)
       {
-          if (ei <= mesh->mlparentsurfaceelement.Size())
+        if (ei < mesh->mlparentsurfaceelement.Size())
-              return mesh->mlparentsurfaceelement.Get(ei)-1;
+          return mesh->mlparentsurfaceelement[ei];
       }
     else
       {
@@ -1013,68 +1011,28 @@ namespace netgen
    int * const indices, int numind) const
 
   {
-    switch (mesh->GetDimension())
+    Point<3> p(hp[0], 0., 0.);
-      {
+    if(mesh->GetDimension() > 1)
-      case 1:
+      p[1] = hp[1];
-        {
+    if(mesh->GetDimension() == 3)
-          Point<3> p(hp[0], 0,0);
+      p[2] = hp[2];
-          for (SegmentIndex si = 0; si < mesh->GetNSeg(); si++)
-            {
-              auto & seg = (*mesh)[si];
-              Point<3> p1 = (*mesh)[seg[0]];
-              Point<3> p2 = (*mesh)[seg[1]];
-              double lam = (p(0)-p1(0)) / (p2(0)-p1(0));
-              if (lam >= -1e-10 && lam <= 1+1e-10)
-                {
-                  lami[0] = 1-lam;
-                  return si;
-                }
-            }
-        }
-        break;
-      case 2:
-        {
-          Point<3> p(hp[0], hp[1],0);
-          try
-            {
-              auto ind = mesh->GetSurfaceElementOfPoint(p, lami, nullptr,
-                                                        build_searchtree);
-              return ind - 1;
-            }
-          catch(NgException e) // quads not implemented curved yet
-            {
-          for (SegmentIndex si = 0; si < mesh->GetNSeg(); si++)
-            {
-              auto & seg = (*mesh)[si];
-              Point<3> p1 = (*mesh)[seg[0]];
-              Point<3> p2 = (*mesh)[seg[1]];
-              double lam;
-              double r;
-              if (fabs(p2[0]-p1[0]) >= fabs(p2[1]-p1[1]))
-                {
-                  lam = (p[0]-p1[0])/(p2[0]-p1[0]);
-                  r = p[1] - p1[1] - lam*(p2[1]-p1[1]);
-                }
-              else
-                {
-                  lam = (p[1]-p1[1])/(p2[1]-p1[1]);
-                  r = p[0] - p1[0] - lam*(p2[0]-p1[0]);
-                }
-              if ( lam >= -1e-10 && lam <= 1+1e-10 && fabs(r) <= 1e-10 )
-                {
-                  lami[0] = 1-lam;
-                  return si;
-                }
-            }
-            }
-        }
-        break;
-      case 3:
-      default:
-        throw Exception("FindElementOfPoint<1> only implemented for mesh-dimension 1 and 2!");
-        break;
-      }
 
+    for (SegmentIndex si = 0; si < mesh->GetNSeg(); si++)
+      {
+        auto & seg = (*mesh)[si];
+        Point<3> p1 = (*mesh)[seg[0]];
+        Point<3> p2 = (*mesh)[seg[1]];
+        Vec<3> v1 = p2-p1;
+        Vec<3> v2 = p-p1;
+        double lam = v1*v2 / v1.Length2();
+        double lam2 = (v2 - lam * v1).Length() / v1.Length();
+
+        if (lam >= -1e-10 && lam <= 1+1e-10 && lam2 < 1e-10)
+          {
+            lami[0] = 1-lam;
+            return si;
+          }
+      }
     return -1;
   }
 
@@ -1085,26 +1043,15 @@ namespace netgen
    int * const indices, int numind) const
 
   {
-    NgArray<int> dummy(numind);
+    Point<3> pp(p[0], p[1], 0.);
-    for (int i = 0; i < numind; i++) dummy[i] = indices[i]+1;
+    if(mesh->GetDimension() == 3)
-    
+      pp[2] = p[2];
+    FlatArray<int> ind(numind, indices);
     double lam3[3];
-    int ind;
+    auto elnr = mesh->GetSurfaceElementOfPoint(pp, lam3, ind, build_searchtree);
-
+    if(elnr.IsValid())
-    if (mesh->GetDimension() == 2)
       {
-        Point<3> p2d(p[0], p[1], 0);
+        if((*mesh)[elnr].GetType() == QUAD || (*mesh)[elnr].GetType() == TRIG6)
-        ind = mesh->GetElementOfPoint(p2d, lam3, &dummy, build_searchtree);
-      }
-    else
-      {
-        Point3d p3d(p[0], p[1], p[2]);
-        ind = mesh->GetSurfaceElementOfPoint(p3d, lam3, &dummy, build_searchtree);
-      }
-    
-    if (ind > 0)
-      {
-        if(mesh->SurfaceElement(ind).GetType()==QUAD || mesh->SurfaceElement(ind).GetType()==TRIG6)
           {
             lami[0] = lam3[0];
             lami[1] = lam3[1];
@@ -1115,7 +1062,7 @@ namespace netgen
             lami[1] = lam3[0];
           }
       }
-    return ind-1;
+    return elnr;
   }
 
 
@@ -1126,13 +1073,9 @@ namespace netgen
    int * const indices, int numind) const
 
   {
-    NgArray<int> dummy(numind);
+    Point<3> pp(p[0], p[1], p[2]);
-    for (int i = 0; i < numind; i++) dummy[i] = indices[i]+1;
+    FlatArray<int> ind(numind, indices);
-    
+    return mesh->GetElementOfPoint(pp, lami, ind, build_searchtree);
-    Point<3> p3d(p[0], p[1], p[2]);
-    int ind = 
-      mesh->GetElementOfPoint(p3d, lami, &dummy, build_searchtree);
-    return ind-1;
   }
 
   void Ngx_Mesh :: Curve (int order)
@@ -1234,7 +1177,7 @@ namespace netgen
   {
     NgLock meshlock (mesh->MajorMutex(), true);
     Refinement & ref = const_cast<Refinement&> (mesh->GetGeometry()->GetRefinement());
-    ::netgen::HPRefinement (*mesh, &ref, SPLIT_ALFELD, 1, 0.5, true, true);
+    ::netgen::HPRefinement (*mesh, &ref, SPLIT_ALFELD, 1, 1.0/3.0, true, true);
   }
 
   
@@ -1249,9 +1192,15 @@ int Ngx_Mesh::GetElementOrder (int enr) const
 void Ngx_Mesh::GetElementOrders (int enr, int * ox, int * oy, int * oz) const
 {
   if (mesh->GetDimension() == 3)
-    mesh->VolumeElement(enr).GetOrder(*ox, *oy, *oz);
+    {
+      ElementIndex ei = IndexBASE<ElementIndex>() + enr-1;
+      mesh->VolumeElement(ei).GetOrder(*ox, *oy, *oz);
+    }
   else
-    mesh->SurfaceElement(enr).GetOrder(*ox, *oy, *oz);
+    {
+      SurfaceElementIndex sei = IndexBASE<SurfaceElementIndex>() + enr-1;      
+      mesh->SurfaceElement(sei).GetOrder(*ox, *oy, *oz);
+    }
 }
 
 void Ngx_Mesh::SetElementOrder (int enr, int order)
@@ -1297,6 +1246,36 @@ int Ngx_Mesh::GetClusterRepElement (int pi) const
 }
 
 
+int Ngx_Mesh::GetElement_Faces (int elnr, int * faces, int * orient) const
+{
+  const MeshTopology & topology = mesh->GetTopology();
+  if (mesh->GetDimension() == 3)
+    {
+      int num = topology.GetElementFaces (elnr+1, faces, orient);
+      for (int i = 0; i < num; i++)
+        faces[i]--;
+      return num;
+    }
+  else
+    {
+      faces[0] = elnr;
+      if (orient) orient[0] = 0;
+      return 1;
+    }
+}
+
+int Ngx_Mesh::GetSurfaceElement_Face (int selnr, int * orient) const
+{
+  if (mesh->GetDimension() == 3)
+    {
+      const MeshTopology & topology = mesh->GetTopology();
+      if (orient)
+	*orient = topology.GetSurfaceElementFaceOrientation (selnr+1);
+      return topology.GetFace (SurfaceElementIndex(selnr));
+    }
+  return -1;
+}
+
 
 
 //HERBERT: falsche Anzahl von Argumenten

libsrc/interface/readuser.cpp

@@ -301,7 +301,7 @@ namespace netgen
                     in >> name;
                     cout << IM(3)  << len << " element are in group " << name << endl;
                     int hi, index;
-                    int fdnr, ednr;
+                    int fdnr=-1, ednr=-1;
 
                     in >> hi >> index >> hi >> hi;
                     int codim = get<1>(element_map[index]);
@@ -712,7 +712,7 @@ namespace netgen
     if(!UserFormatRegister::HaveFormat(format))
       throw Exception("Unknown format: " + format);
 
-    const auto & entry = UserFormatRegister::Get(format);
+    const auto entry = UserFormatRegister::Get(format);
     if(!entry.read)
       throw Exception("Reading format " + format + " is not implemented");
 

libsrc/interface/rw_cgns.cpp

@@ -8,7 +8,7 @@
 
 namespace netgen::cg
 {
-  typedef ngcore::ClosedHashTable<ngcore::INT<3,size_t>, size_t> PointTable;
+  typedef ngcore::ClosedHashTable<ngcore::IVec<3,size_t>, size_t> PointTable;
 
   int getDim(ElementType_t type)
     {
@@ -416,7 +416,7 @@ namespace netgen::cg
 
           for(auto i : Range(nv))
           {
-            ngcore::INT<3,size_t> hash = {*reinterpret_cast<size_t*>(&x[i]), *reinterpret_cast<size_t*>(&y[i]), *reinterpret_cast<size_t*>(&z[i])};
+            ngcore::IVec<3,size_t> hash = {*reinterpret_cast<size_t*>(&x[i]), *reinterpret_cast<size_t*>(&y[i]), *reinterpret_cast<size_t*>(&z[i])};
             size_t pi_ng;
             size_t pos;
             // check if this point is new

libsrc/interface/rw_medit.cpp

@@ -50,7 +50,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int nvert;
       fin >> nvert;
       Point<3> p{0.,0.,0.};
-      for(auto k : Range(nvert)) {
+      for([[maybe_unused]] auto k : Range(nvert)) {
         for(auto i : Range(dim))
           fin >> p[i];
         fin >> index;
@@ -61,7 +61,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int nedge;
       fin >> nedge;
       Segment seg;
-      for(auto k : Range(nedge)) {
+      for([[maybe_unused]] auto k : Range(nedge)) {
         for(auto i : Range(2))
           fin >> seg[i];
         fin >> seg.edgenr;
@@ -74,7 +74,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int ntrig, index;
       fin >> ntrig;
       Element2d sel;
-      for(auto k : Range(ntrig)) {
+      for([[maybe_unused]] auto k : Range(ntrig)) {
         for(auto i : Range(3))
           fin >> sel[i];
         fin >> index;
@@ -86,7 +86,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int ntet;
       fin >> ntet;
       Element el(4);
-      for(auto k : Range(ntet)) {
+      for([[maybe_unused]] auto k : Range(ntet)) {
         for(auto i : Range(4))
           fin >> el[i];
         fin >> index;
@@ -99,7 +99,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int ncorners;
       fin >> ncorners;
       Element0d el;
-      for(auto k : Range(ncorners)) {
+      for([[maybe_unused]] auto k : Range(ncorners)) {
           fin >> el.pnum;
       }
     }
@@ -107,7 +107,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int nverts;
       fin >> nverts;
       int vert;
-      for(auto k : Range(nverts)) {
+      for([[maybe_unused]] auto k : Range(nverts)) {
           fin >> vert;
       }
     }
@@ -115,7 +115,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int nnormals;
       fin >> nnormals;
       Vec<3> normal;
-      for(auto k : Range(nnormals)) {
+      for([[maybe_unused]] auto k : Range(nnormals)) {
           fin >> normal[0];
           fin >> normal[1];
           fin >> normal[2];
@@ -126,7 +126,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       fin >> nnormals;
       int vert;
       int normal;
-      for(auto k : Range(nnormals)) {
+      for([[maybe_unused]] auto k : Range(nnormals)) {
         fin >> normal;
         fin >> vert;
       }
@@ -135,7 +135,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int ntangents;
       fin >> ntangents;
       Vec<3> tangent;
-      for(auto k : Range(ntangents)) {
+      for([[maybe_unused]] auto k : Range(ntangents)) {
         fin >> tangent[0];
         fin >> tangent[1];
         fin >> tangent[2];
@@ -146,7 +146,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       fin >> ntangents;
       int vert;
       int tangent;
-      for(auto k : Range(ntangents)) {
+      for([[maybe_unused]] auto k : Range(ntangents)) {
         fin >> tangent;
         fin >> vert;
       }
@@ -155,7 +155,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int nridges;
       fin >> nridges;
       int ridge;
-      for(auto k : Range(nridges)) {
+      for([[maybe_unused]] auto k : Range(nridges)) {
         fin >> ridge;
       }
     }
@@ -164,7 +164,7 @@ void ReadMeditFormat (Mesh & mesh, const filesystem::path & filename, map<tuple<
       int nitems;
       fin >> nitems;
       string s;
-      for(auto i : Range(nitems))
+      for([[maybe_unused]] auto i : Range(nitems))
         fin >> s; // read one line
     }
   }

libsrc/interface/writeOpenFOAM15x.cpp

@@ -215,7 +215,8 @@ namespace netgen
             // Check if the face is a surface element (boundary face)
             // if not, add the current volume element and the corresponding face into 
             // the owner list
-            int surfelem = meshtopo.GetFace2SurfaceElement(absfacenr);
+            // int surfelem = meshtopo.GetFace2SurfaceElement1(absfacenr);
+            int surfelem = meshtopo.GetFace2SurfaceElement(absfacenr-1)+1;
             if(!surfelem)
             {
                // If it is a new face which has not been listed before, 
@@ -606,7 +607,7 @@ namespace netgen
       const_cast<Mesh&> (mesh).Compress();
       const_cast<Mesh&> (mesh).CalcSurfacesOfNode();
       const_cast<Mesh&> (mesh).RebuildSurfaceElementLists();
-      const_cast<Mesh&> (mesh).BuildElementSearchTree();
+      const_cast<Mesh&> (mesh).BuildElementSearchTree(3);
 
 
       int np = mesh.GetNP();

libsrc/interface/writeabaqus.cpp

@@ -15,15 +15,103 @@
 namespace netgen
 {
 
+using std::vector;
 
+struct AbaqusElementType
+{
+  const char * name;
+  const vector<int> permutation;
 
+  AbaqusElementType(const char * name, const vector<int> & permutation)
+    : name(name), permutation(permutation)
+  {}
+};
+
+static inline const AbaqusElementType & GetAbaqusType(int dim, int num_nodes)
+{
+  // maps num_nodes to AbaqusElementType for each dimension
+  typedef std::map<int, AbaqusElementType> AbaqusElementTypes;
+  static const std::map<int, AbaqusElementType> abaqus_eltypes[3] =
+  {
+    // 1D
+    AbaqusElementTypes{
+      {2, AbaqusElementType{"T2D2", vector{0,1}}},
+    },
+    // 2D
+    AbaqusElementTypes{
+      {3, AbaqusElementType{"CPS3", vector{0,1,2}}},
+    },
+    // 3D
+    AbaqusElementTypes{
+      {4, AbaqusElementType{"C3D4", vector{0,1,3,2}}},
+      {10, AbaqusElementType{"C3D10", vector{0,1,3,2,4,8,6,5,7,9}}},
+    }
+  };
+
+  const auto & eltypes = abaqus_eltypes[dim-1];
+  if (eltypes.count(num_nodes) > 0)
+    return eltypes.at(num_nodes);
+  else
+    throw Exception("unsupported " + ToString(dim)+"d Element type with " + ToString(num_nodes) + " nodes");
+}
+
+static void WritePoints ( const Mesh & mesh, ostream & out )
+{
+  out << "*Node" << endl;
+  for(auto pi : mesh.Points().Range() )
+  {
+    out << pi+1-IndexBASE<PointIndex>() << ", ";
+    auto p = mesh[pi];
+    out << p[0] << ", " << p[1] << ", " << p[2] << '\n';
+  }
+}
+
+template <typename ElIndex>
+static void WriteElement(ostream & out, const Mesh& mesh, ElIndex ei, const vector<int> & permutation, int & el_counter)
+{
+  el_counter++;
+  auto el = mesh[ei];
+  out << el_counter;
+  for(auto i : Range(el.PNums()))
+    out << ", " << el[permutation[i]]+1-IndexBASE<PointIndex>();
+  out << '\n';
+}
+
+template <typename ElIndex, typename Elements>
+static void WriteElements ( ostream & out, const Mesh & mesh, int dim, const Elements & el_range, int & el_counter)
+{
+  // map index, num_nodes to elements
+  std::map<std::tuple<int, int>, Array<ElIndex>> elset_map;
+
+  for(auto ei : el_range)
+    {
+      const auto & el = mesh[ei];
+      int index = 0;
+      if constexpr(std::is_same_v<ElIndex,SegmentIndex>)
+        index = el.edgenr;
+      else
+        index = el.GetIndex();
+      elset_map[{index, el.GetNP()}].Append(ei);
+    }
+
+  for(auto & [key, elems] : elset_map)
+    {
+      auto [index, num_nodes] = key;
+      auto name = mesh.GetRegionName(elems[0]);
+      if (name == "") name = "default";
+      PrintMessage (5, index, ": ", name);
+      const auto & eltype = GetAbaqusType(dim, num_nodes) ;
+      out << "*Element, type=" << eltype.name << ", ELSET=" << name << endl;
+      for(auto ei : elems)
+        WriteElement(out, mesh, ei, eltype.permutation, el_counter);
+    }
+}
 
 void WriteAbaqusFormat (const Mesh & mesh,
 			const filesystem::path & filename)
 
 {
-      
+  PrintMessage (1, "Write Abaqus Mesh");
-  cout << "\nWrite Abaqus Volume Mesh" << endl;
 
   ofstream outfile (filename);
 
@@ -32,96 +120,17 @@ void WriteAbaqusFormat (const Mesh & mesh,
 
   outfile.precision(8);
 
-  outfile << "*Node" << endl;
+  int element_counter = 0;
-
+  WritePoints(mesh, outfile);
-  int np = mesh.GetNP();
+  if(mesh.GetDimension() < 3)
-  int ne = mesh.GetNE();
+    WriteElements<SegmentIndex>(outfile, mesh, 1, mesh.LineSegments().Range(), element_counter);
-  int i, j, k;
+  WriteElements<SurfaceElementIndex>(outfile, mesh, 2, mesh.SurfaceElements().Range(), element_counter);
-
+  WriteElements<ElementIndex>(outfile, mesh, 3, mesh.VolumeElements().Range(), element_counter);
-  for (i = 1; i <= np; i++)
-    {
-      outfile << i << ", ";
-      outfile << mesh.Point(i)(0) << ", ";
-      outfile << mesh.Point(i)(1) << ", ";
-      outfile << mesh.Point(i)(2) << "\n";
-    }
-
-  int elemcnt = 0; //element counter
-  int finished = 0;
-  int indcnt = 1; //index counter
-
-  while (!finished)
-    {
-      int actcnt = 0;
-      const Element & el1 = mesh.VolumeElement(1);
-      int non = el1.GetNP();
-      if (non == 4)
-	{
-	  outfile << "*Element, type=C3D4, ELSET=PART" << indcnt << endl;
-	} 
-      else if (non == 10)
-	{
-	  outfile << "*Element, type=C3D10, ELSET=PART" << indcnt << endl;
-	} 
-      else
-	{
-	  cout << "unsupported Element type!!!" << endl;	  
-	}
-
-      for (i = 1; i <= ne; i++)
-	{
-	  const Element & el = mesh.VolumeElement(i);
-	      
-	  if (el.GetIndex() == indcnt)
-	    {
-	      actcnt++;
-	      if (el.GetNP() != non) 
-		{
-		  cout << "different element-types in a subdomain are not possible!!!" << endl;
-		  continue;
-		}
-		  
-	      elemcnt++;
-	      outfile << elemcnt << ", ";
-	      if (non == 4)
-		{
-		  outfile << el.PNum(1) << ", ";
-		  outfile << el.PNum(2) << ", ";
-		  outfile << el.PNum(4) << ", ";
-		  outfile << el.PNum(3) << "\n";
-		}
-	      else if (non == 10)
-		{
-		  outfile << el.PNum(1) << ", ";
-		  outfile << el.PNum(2) << ", ";
-		  outfile << el.PNum(4) << ", ";
-		  outfile << el.PNum(3) << ", ";
-		  outfile << el.PNum(5) << ", ";
-		  outfile << el.PNum(9) << ", ";
-		  outfile << el.PNum(7) << ", " << "\n";
-		  outfile << el.PNum(6) << ", ";
-		  outfile << el.PNum(8) << ", ";
-		  outfile << el.PNum(10) << "\n";
-		}
-	      else
-		{
-		  cout << "unsupported Element type!!!" << endl;
-		  for (j = 1; j <= el.GetNP(); j++)
-		    {
-		      outfile << el.PNum(j);
-		      if (j != el.GetNP()) outfile << ", ";
-		    }
-		  outfile << "\n";
-		}
-	    }
-	}	  
-      indcnt++;
-      if (elemcnt == ne) {finished = 1; cout << "all elements found by Index!" << endl;}
-      if (actcnt == 0) {finished = 1;}
-    }
 
+  // Write identifications (untested!)
   if (mesh.GetIdentifications().GetMaxNr())
     {
+      const auto np = mesh.GetNP();
       // periodic identification, implementation for
       // Helmut J. Boehm, TU Vienna
 	  
@@ -138,27 +147,27 @@ void WriteAbaqusFormat (const Mesh & mesh,
       NgArray<INDEX_2> pairs;
       NgBitArray master(np), help(np);
       master.Set();
-      for (i = 1; i <= 3; i++)
+      for (int i = 1; i <= 3; i++)
 	{
 	  mesh.GetIdentifications().GetPairs (i, pairs);
 	  help.Clear();
-	  for (j = 1; j <= pairs.Size(); j++)
+	  for (int j = 1; j <= pairs.Size(); j++)
 	    {
 	      help.Set (pairs.Get(j).I1());
 	    }
 	  master.And (help);
 	}
-      for (i = 1; i <= np; i++)
+      for (int i = 1; i <= np; i++)
 	if (master.Test(i))
 	  masternode = i;
 
       cout << "masternode = " << masternode << " = "
 	   << mesh.Point(masternode) << endl;
       NgArray<int> minions(3);
-      for (i = 1; i <= 3; i++)
+      for (int i = 1; i <= 3; i++)
 	{
 	  mesh.GetIdentifications().GetPairs (i, pairs);
-	  for (j = 1; j <= pairs.Size(); j++)
+	  for (int j = 1; j <= pairs.Size(); j++)
 	    {
 	      if (pairs.Get(j).I1() == masternode)
 		minions.Elem(i) = pairs.Get(j).I2();
@@ -179,12 +188,12 @@ void WriteAbaqusFormat (const Mesh & mesh,
 	      << "**POINT_fixed\n"
 	      << "**\n"
 	      << "*BOUNDARY, OP=NEW\n";
-      for (j = 1; j <= 3; j++)
+      for (int j = 1; j <= 3; j++)
 	outfile << masternode << ", " << j << ",,    0.\n";
 
       outfile << "**\n"
 	      << "*BOUNDARY, OP=NEW\n";
-      for (j = 1; j <= 3; j++)
+      for (int j = 1; j <= 3; j++)
 	{
 	  Vec3d v(mesh.Point(masternode), mesh.Point(minions.Get(j)));
 	  double vlen = v.Length();
@@ -203,18 +212,18 @@ void WriteAbaqusFormat (const Mesh & mesh,
 
       NgBitArray eliminated(np);
       eliminated.Clear();
-      for (i = 1; i <= mesh.GetIdentifications().GetMaxNr(); i++)
+      for (int i = 1; i <= mesh.GetIdentifications().GetMaxNr(); i++)
 	{
 	  mesh.GetIdentifications().GetPairs (i, pairs);
 	  if (!pairs.Size())
 	    continue;
 	      
-	  for (j = 1; j <= pairs.Size(); j++)
+	  for (int j = 1; j <= pairs.Size(); j++)
 	    if (pairs.Get(j).I1() != masternode && 
 		!eliminated.Test(pairs.Get(j).I2()))
 	      {
 		eliminated.Set (pairs.Get(j).I2());
-		for (k = 1; k <= 3; k++)
+		for (int k = 1; k <= 3; k++)
 		  {
 		    mpc << "4" << "\n";
 		    mpc << pairs.Get(j).I2() << "," << k << ", -1.0, ";
@@ -227,7 +236,7 @@ void WriteAbaqusFormat (const Mesh & mesh,
     }
 
 
-  cout << "done" << endl;
+  PrintMessage(1, "done");
 }
 
 static RegisterUserFormat reg_abaqus ("Abaqus Format", {".mesh"}, nullopt, WriteAbaqusFormat);