return invalid surfaceindex (index is 0 based)

Don't check for intersecting mapped trigs if only volume elements are
added (and no mapped surface elements).

In case of only inserted volume elements, we don't care about the limit
at special points.

.clang-format

@@ -0,0 +1,64 @@
+Language: Cpp
+BasedOnStyle: LLVM
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands: true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: InlineOnly
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterClass: true
+  AfterControlStatement: true
+  AfterEnum: true
+  AfterFunction: true
+  AfterNamespace: true
+  AfterStruct: true
+  AfterUnion: true
+  BeforeCatch: true
+  BeforeElse: true
+  IndentBraces: true
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakInheritanceList: AfterColon
+ColumnLimit: 0
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 2
+ContinuationIndentWidth: 2
+Cpp11BracedListStyle: true
+EmptyLineBeforeAccessModifier: Never
+IndentCaseLabels: false
+IndentPPDirectives: None
+IndentWidth: 2
+KeepEmptyLinesAtTheStartOfBlocks: false
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+PointerAlignment: Left
+ReflowComments: true
+SortIncludes: false
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceBeforeParens: Custom
+SpaceBeforeParensOptions:
+  AfterControlStatements: true
+  AfterFunctionDefinitionName: true
+  AfterFunctionDeclarationName: true
+SpacesInAngles: false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInSquareBrackets: false
+Standard: Latest
+TabWidth: 2
+UseTab: Never
+

.gitignore

@@ -0,0 +1,11 @@
+*.whl
+dist
+build
+*.vol.gz
+*.vol
+*.ini
+__pycache__
+*.json
+*.zip
+.cache
+*.patch

.gitlab-ci.yml

@@ -2,21 +2,21 @@ stages:
   - build
   - test
   - test_ngsolve
+  - deploy
   - cleanup
 
-push_github_sourceforge:
+push_github:
   stage: build
   tags:
     - linux
     - docker
+    - bash
   script:
-    - git remote add sourceforge ssh://mhochste@git.code.sf.net/p/netgen-mesher/git || true
     - git remote add github git@github.com:NGSolve/netgen.git || true
     - git remote update
-    - git checkout master
+    - git checkout --track origin/master
     - git pull origin master
-    - git push sourceforge master
-    - git push github master
+    - git push github master --tags
   only:
     - master
 
@@ -30,15 +30,25 @@ push_github_sourceforge:
     - x64
   before_script:
     - "echo off"
-    - 'call "%VS2017INSTALLDIR%\VC\Auxiliary\Build\vcvars64"'
+    - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64"
     - set CI_DIR=C:\ci\%CI_PIPELINE_ID%
-    - set CLCACHE_BASEDIR=C:\ci\%CI_PIPELINE_ID%
+    - set CCACHE_BASEDIR=C:\ci\%CI_PIPELINE_ID%
     - set NETGEN_BUILD_DIR=%CI_DIR%\build
     - set INSTALL_DIR=%CI_DIR%\install
     - set SRC_DIR=%CI_DIR%\src
     - set NETGENDIR=%INSTALL_DIR%\bin
     - set PYTHONPATH=%INSTALL_DIR%\lib\site-packages
-    - set PATH=%NETGENDIR%;%PATH%
+    - echo %PATH%
+    - set PATH=%INSTALL_DIR%\bin;C:\python312;C:\python312\bin;C:\python312\Scripts;C:\tools\;%PATH%
+    - echo %PATH%
+    - set CCACHE_HARDLINK=1
+    - set CCACHE_NOHASHDIR=1
+    - C:\tools\ccache -s
+    - C:\tools\ccache -M 20G
+    - dir C:\python312
+    - python.exe --version
+    - python.exe -m pip install -U netgen-occt netgen-occt-devel
+    - cmake --version
 
 build_win:
   <<: *win
@@ -54,14 +64,14 @@ build_win:
     - >-
         cmake %SRC_DIR%
         -G Ninja
+        -DCMAKE_PREFIX=C:/python312
+        -DPython3_ROOT_DIR=C:/python312
         -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR%
+        -DCHECK_RANGE=ON
         -DUSE_CGNS=ON
         -DUSE_OCC=ON
-        -DOCC_LIBRARY=C:/install_opencascade_7.4.0_static/win64/vc14/lib/TKernel.lib
-        -DOCC_INCLUDE_DIR=C:/install_opencascade_7.4.0_static/inc
-        -DOCC_LINK_FREETYPE=ON
         -DUSE_CCACHE=ON
-        -DENABLE_UNIT_TESTS=ON
+        -DENABLE_UNIT_TESTS=OFF
         -DCMAKE_BUILD_TYPE=Release
     - cmake --build . --target install --config Release
 
@@ -69,12 +79,28 @@ test_win:
   <<: *win
   stage: test
   script:
+    - pip install pytest-check
     - cd tests\pytest
     - cd %NETGEN_BUILD_DIR%\netgen
     - ctest -C Release -V --output-on-failure
     - cd ..
   needs: ["build_win"]
 
+generate_results:
+  <<: *win
+  stage: test
+  script:
+    - pip install pytest-check
+    - cd tests\pytest
+    - python test_tutorials.py new_results.json
+  needs: ["build_win"]
+  when: manual
+  artifacts:
+    paths:
+      - tests/pytest/new_results.json
+    when: always
+    expire_in: 1 week
+
 cleanup_win:
   <<: *win
   stage: cleanup
@@ -95,12 +121,14 @@ cleanup_win:
 .template_ubuntu: &ubuntu
   tags:
     - linux
+    - bash
   before_script:
     - pwd
     - ls
     - docker info
+    - export PYTHONPATH=/opt/netgen/`python3 -c "import os.path, sysconfig;print(os.path.relpath(sysconfig.get_path('platlib'), sysconfig.get_path('data')))"`
   variables:
-    UBUNTU_VERSION: "18.04"
+    UBUNTU_VERSION: "22.04"
 
 build_ubuntu_debug:
   <<: *ubuntu
@@ -112,6 +140,7 @@ build_ubuntu_debug:
       docker run
       --cidfile netgen_${CI_PIPELINE_ID}_${UBUNTU_VERSION}.id
       -e CCACHE_DIR=/ccache
+      -e PYTHONPATH=$PYTHONPATH
       -v /mnt/ccache:/ccache
       netgen_${CI_PIPELINE_ID}:${UBUNTU_VERSION}
       bash /root/src/netgen/tests/build_debug.sh
@@ -128,6 +157,7 @@ build_ubuntu_mpi:
       docker run>-
       --cidfile netgen_mpi_${CI_PIPELINE_ID}_${UBUNTU_VERSION}.id>-
       -e CCACHE_DIR=/ccache
+      -e PYTHONPATH=$PYTHONPATH
       -e RUN_SLOW_TESTS=${RUN_SLOW_TESTS}
       -v /mnt/ccache:/ccache
       netgen_mpi_${CI_PIPELINE_ID}:${UBUNTU_VERSION}
@@ -141,8 +171,7 @@ test_ubuntu_debug:
   script:
     - >-
       docker run
-      -e NETGENDIR=/opt/netgen/bin
-      -e PYTHONPATH=/opt/netgen/lib/python3/dist-packages
+      -e PYTHONPATH=$PYTHONPATH
       netgen_${CI_PIPELINE_ID}_installed:${UBUNTU_VERSION}
       bash -c 'cd /root/build/netgen && make test_netgen ARGS="-V --output-on-failure"'
   needs: ["build_ubuntu_debug"]
@@ -154,8 +183,7 @@ test_ubuntu_mpi:
     - >-
       docker run
       -e RUN_SLOW_TESTS=${RUN_SLOW_TESTS}
-      -e NETGENDIR=/opt/netgen/bin
-      -e PYTHONPATH=/opt/netgen/lib/python3/dist-packages
+      -e PYTHONPATH=$PYTHONPATH
       netgen_mpi_${CI_PIPELINE_ID}_installed:${UBUNTU_VERSION}
       bash -c 'cd /root/build/netgen && make test_netgen ARGS="-V --output-on-failure"'
   needs: ["build_ubuntu_mpi"]
@@ -168,7 +196,7 @@ test_build_ngsolve:
     - >-
       docker run
       -e NETGENDIR=/opt/netgen/bin
-      -e PYTHONPATH=/opt/netgen/lib/python3/dist-packages
+      -e PYTHONPATH=$PYTHONPATH
       -e MKLROOT=/opt/intel/mkl
       -v /opt/intel:/opt/intel
       -e CCACHE_DIR=/ccache
@@ -176,26 +204,11 @@ test_build_ngsolve:
       netgen_${CI_PIPELINE_ID}_installed:${UBUNTU_VERSION}
       bash -c 'cd /root/src/netgen/tests/ && ./build_ngsolve.sh'
 
-# cpp guideline checks
-# test_guidelines:
-#   <<: *ubuntu
-#   stage: test
-#   script:
-#     - docker run -e CCACHE_DIR=/ccache -v /mnt/ccache:/ccache netgen_${CI_PIPELINE_ID}:${UBUNTU_VERSION} bash /root/src/netgen/tests/build_guidelines.sh
-#   when: always
-#   allow_failure: true
-
-# check if it compiles without spdlog
-test_noSpdlog:
-  <<: *ubuntu
-  stage: test
-  script:
-    - docker run -e CCACHE_DIR=/ccache -v /mnt/ccache:/ccache netgen_${CI_PIPELINE_ID}:${UBUNTU_VERSION} bash /root/src/netgen/tests/build_nospdlog.sh
-
 cleanup_ubuntu:
   stage: cleanup
   tags:
     - linux
+    - bash
   script:
     # remove intermediate and old docker images and containers
     - docker rm -f `docker ps --no-trunc -aq`
@@ -215,12 +228,13 @@ cleanup_ubuntu:
 .template_mac: &mac
   tags:
     - mac
+    - x64
   before_script:
     - export ROOT_DIR=/tmp/$CI_PIPELINE_ID
     - export SRC_DIR=$ROOT_DIR/src
     - export BUILD_DIR=$ROOT_DIR/build
     - export CMAKE_INSTALL_PREFIX=/tmp/$CI_PIPELINE_ID/install/Netgen.app
-    - export PYTHONPATH=$CMAKE_INSTALL_PREFIX/Contents/Resources/`python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1,0,''))"`:.
+    - export PYTHONPATH=$CMAKE_INSTALL_PREFIX/Contents/Resources/`python3 -c "import os.path, sysconfig;print(os.path.relpath(sysconfig.get_path('platlib'), sysconfig.get_path('data')))"`:.
     - export PATH=$CMAKE_INSTALL_PREFIX/Contents/MacOS:$PATH
 
 build_mac:
@@ -238,16 +252,15 @@ build_mac:
         cmake $SRC_DIR
         -DCMAKE_INSTALL_PREFIX=$CMAKE_INSTALL_PREFIX
         -DCMAKE_BUILD_TYPE=Release
+        -DCHECK_RANGE=ON
         -DUSE_NATIVE_ARCH=OFF
         -DUSE_CCACHE=ON
         -DENABLE_UNIT_TESTS=ON
-        -DCMAKE_OSX_DEPLOYMENT_TARGET=10.12
+        -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15
         -DCMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
         -DUSE_CGNS=ON
         -DUSE_OCC=ON
-        -DOCC_LIBRARY=/usr/local/opt/opencascade-7.4.0/lib/libTKernel.a
-        -DOCC_INCLUDE_DIR=/usr/local/opt/opencascade-7.4.0/include/opencascade
-        -DOCC_LINK_FREETYPE=ON
+        -DPython3_ROOT_DIR=/Library/Frameworks/Python.framework/Versions/3.8/
     - make -j5 install
 
 test_mac:
@@ -267,3 +280,40 @@ cleanup_mac:
   allow_failure: true
   needs: ["test_mac"]
 
+pip_linux:
+  image: quay.io/pypa/manylinux_2_28_x86_64
+  stage: build
+  tags:
+    - pip
+    - linux
+    - docker
+  script:
+    - ./tests/build_pip.sh
+  when: manual
+
+pip_windows:
+  stage: build
+  tags:
+    - pip
+    - windows
+  script:
+    - .\tests\build_pip.ps1 C:\Python313
+    - .\tests\build_pip.ps1 C:\Python312
+    - .\tests\build_pip.ps1 C:\Python311
+    - .\tests\build_pip.ps1 C:\Python310
+    - .\tests\build_pip.ps1 C:\Python39
+  when: manual
+
+pip_macos:
+  stage: build
+  tags:
+    - pip
+    - macosx
+    - m1
+  script:
+    - ./tests/build_pip_mac.sh 3.13
+    - ./tests/build_pip_mac.sh 3.12
+    - ./tests/build_pip_mac.sh 3.11
+    - ./tests/build_pip_mac.sh 3.10
+    - ./tests/build_pip_mac.sh 3.9
+  when: manual

CMakeLists.txt

@@ -2,19 +2,26 @@ if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING INTERNAL)
 endif(NOT CMAKE_BUILD_TYPE)
 
-if(WIN32)
-  # we are linking to object libraries on Windows
-  cmake_minimum_required(VERSION 3.12)
-else(WIN32)
-  cmake_minimum_required(VERSION 3.8)
-endif(WIN32)
+cmake_minimum_required(VERSION 3.16)
+cmake_policy(VERSION 3.16)
 
+if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0")
+  cmake_policy(SET CMP0135 NEW)
+endif()
+
+include (CMakeDependentOption)
 option( USE_NATIVE_ARCH  "build for native cpu architecture" ON)
 
-option( USE_GUI     "don't build netgen with GUI" ON )
+option( USE_GUI     "build with GUI" ON )
 option( USE_PYTHON  "build with python interface" ON )
+cmake_dependent_option( PREFER_SYSTEM_PYBIND11  "Use system wide PyBind11" OFF "USE_PYTHON" OFF)
 option( USE_MPI     "enable mpi parallelization" OFF )
-option( USE_OCC     "(not supported) compile with OpenCascade geometry kernel" OFF)
+option( USE_MPI_WRAPPER "enable mpi wrapper (run-time dispatch of MPI library calls)" OFF )
+option( USE_OCC     "build with OpenCascade geometry kernel interface" ON)
+option( USE_STLGEOM     "build with STL geometry support" ON)
+option( USE_CSG     "build with CSG kernel" ON)
+option( USE_INTERFACE     "build nginterface" ON)
+option( USE_GEOM2D     "build 2d geometry kernels" ON)
 option( USE_JPEG    "enable snapshots using library libjpeg" OFF )
 option( USE_MPEG    "enable video recording with FFmpeg, uses libavcodec" OFF )
 option( USE_CGNS    "enable CGNS file read/write support" OFF )
@@ -25,13 +32,19 @@ option( USE_CCACHE       "use ccache")
 option( USE_INTERNAL_TCL "Compile tcl files into the code and don't install them" ON)
 option( ENABLE_UNIT_TESTS "Enable Catch unit tests")
 option( ENABLE_CPP_CORE_GUIDELINES_CHECK "Enable cpp core guideline checks on ngcore" OFF)
-option( USE_SPDLOG "Enable spd log logging" OFF)
 option( DEBUG_LOG "Enable more debug output (may increase computation time) - only works with USE_SPDLOG=ON" OFF)
 option( CHECK_RANGE "Check array range access, automatically enabled if built in debug mode" OFF)
 option( BUILD_STUB_FILES "Build stub files for better autocompletion" ON)
 option( BUILD_FOR_CONDA "Link python libraries only to executables" OFF)
 
-option( USE_SUPERBUILD       "use ccache" ON)
+option( USE_SUPERBUILD       "build dependencies automatically" ON)
+option( TRACE_MEMORY     "Enable memory tracing" OFF)
+
+set(NG_COMPILE_FLAGS "" CACHE STRING "Additional compile flags")
+
+set(NGLIB_LIBRARY_TYPE SHARED CACHE STRING "nglib library type")
+set(NGCORE_LIBRARY_TYPE SHARED CACHE STRING "ngcore library type")
+set(NGGUI_LIBRARY_TYPE SHARED CACHE STRING "nggui library type")
 
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/cmake_modules")
 
@@ -50,30 +63,94 @@ if(INSTALL_DIR)
   set(INSTALL_DIR_DEFAULT ${INSTALL_DIR})
 endif(INSTALL_DIR)
 
-if(UNIX)
+if(UNIX AND USE_SUPERBUILD)
   message("Checking for write permissions in install directory...")
   execute_process(COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX})
   execute_process(COMMAND test -w ${CMAKE_INSTALL_PREFIX} RESULT_VARIABLE res)
   if(res)
     message(WARNING "No write access at install directory, please set correct permissions")
   endif()
-endif(UNIX)
+endif(UNIX AND USE_SUPERBUILD)
 
 if (USE_SUPERBUILD)
   project (SUPERBUILD)
-  if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-    set(CMAKE_INSTALL_PREFIX "${INSTALL_DIR_DEFAULT}" CACHE PATH "Install directory" FORCE)
+else()
+  project(Netgen)
+endif()
+
+if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX "${INSTALL_DIR_DEFAULT}" CACHE PATH "Install directory" FORCE)
+endif()
+
+set(NG_INSTALL_SUFFIX netgen CACHE STRING "Suffix appended to install directories (project name)")
+
+if(USE_PYTHON)
+  if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.18)
+      find_package(Python3 REQUIRED COMPONENTS Development.Module)
+      if(NOT EMSCRIPTEN)
+          find_package(Python3 COMPONENTS Interpreter Development.Embed)
+      endif()
+  else()
+    find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
   endif()
 
+  if(NOT CMAKE_CROSSCOMPILING)
+      find_package(Python3 REQUIRED COMPONENTS Interpreter)
+      execute_process(COMMAND ${Python3_EXECUTABLE} -c "import os.path, sysconfig;print(os.path.relpath(sysconfig.get_path('platlib'), sysconfig.get_path('data')))" OUTPUT_VARIABLE PYTHON_PACKAGES_INSTALL_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
+      file(TO_CMAKE_PATH ${PYTHON_PACKAGES_INSTALL_DIR} PYTHON_PACKAGES_INSTALL_DIR)
+  endif(NOT CMAKE_CROSSCOMPILING)
+endif(USE_PYTHON)
+
+if(APPLE AND NOT EMSCRIPTEN)
+  set(NG_INSTALL_DIR_BIN_DEFAULT Contents/MacOS)
+  set(NG_INSTALL_DIR_LIB_DEFAULT Contents/MacOS)
+  set(NG_INSTALL_DIR_CMAKE_DEFAULT Contents/Resources/CMake)
+  set(NG_INSTALL_DIR_PYTHON_DEFAULT Contents/Resources/${PYTHON_PACKAGES_INSTALL_DIR})
+  set(NG_INSTALL_DIR_RES_DEFAULT Contents/Resources/share)
+  set(NG_INSTALL_DIR_INCLUDE_DEFAULT Contents/Resources/include)
+
+  set(NG_RPATH_TOKEN "@loader_path")
+else(APPLE AND NOT EMSCRIPTEN)
+  set(NG_INSTALL_DIR_BIN_DEFAULT bin)
+  set(NG_INSTALL_DIR_LIB_DEFAULT lib)
+  if(WIN32)
+    set(NG_INSTALL_DIR_CMAKE_DEFAULT cmake)
+  else(WIN32)
+    set(NG_INSTALL_DIR_CMAKE_DEFAULT lib/cmake/${NG_INSTALL_SUFFIX})
+  endif(WIN32)
+  set(NG_INSTALL_DIR_PYTHON_DEFAULT ${PYTHON_PACKAGES_INSTALL_DIR})
+  set(NG_INSTALL_DIR_RES_DEFAULT share)
+  set(NG_INSTALL_DIR_INCLUDE_DEFAULT include)
+
+  set(NG_RPATH_TOKEN "\$ORIGIN")
+endif(APPLE AND NOT EMSCRIPTEN)
+
+set(NG_INSTALL_DIR_PYTHON ${NG_INSTALL_DIR_PYTHON_DEFAULT} CACHE STRING "Install directory for Python files")
+set(NG_INSTALL_DIR_BIN ${NG_INSTALL_DIR_BIN_DEFAULT} CACHE STRING "Install directory for executables")
+set(NG_INSTALL_DIR_LIB ${NG_INSTALL_DIR_LIB_DEFAULT} CACHE STRING "Install directory for libraries")
+set(NG_INSTALL_DIR_INCLUDE ${NG_INSTALL_DIR_INCLUDE_DEFAULT} CACHE STRING "Install directory for header files")
+set(NG_INSTALL_DIR_CMAKE ${NG_INSTALL_DIR_CMAKE_DEFAULT} CACHE STRING "Install directory for CMake files")
+set(NG_INSTALL_DIR_RES ${NG_INSTALL_DIR_RES_DEFAULT} CACHE STRING "Install directory for resources")
+
+get_filename_component(NETGEN_CMAKE_DIR_ABSOLUTE ${NG_INSTALL_DIR_CMAKE} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
+get_filename_component(NETGEN_BINARY_DIR_ABSOLUTE ${NG_INSTALL_DIR_BIN} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
+get_filename_component(NETGEN_LIBRARY_DIR_ABSOLUTE ${NG_INSTALL_DIR_LIB} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
+get_filename_component(NETGEN_INCLUDE_DIR_ABSOLUTE ${NG_INSTALL_DIR_INCLUDE} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
+get_filename_component(NETGEN_RESOURCE_DIR_ABSOLUTE ${NG_INSTALL_DIR_RES} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
+
+file(RELATIVE_PATH NETGEN_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${CMAKE_INSTALL_PREFIX})
+file(RELATIVE_PATH NETGEN_BINARY_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_BINARY_DIR_ABSOLUTE})
+file(RELATIVE_PATH NETGEN_LIBRARY_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_LIBRARY_DIR_ABSOLUTE})
+file(RELATIVE_PATH NETGEN_INCLUDE_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_INCLUDE_DIR_ABSOLUTE})
+file(RELATIVE_PATH NETGEN_RESOURCE_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_RESOURCE_DIR_ABSOLUTE})
+
+file(RELATIVE_PATH NETGEN_RPATH ${NETGEN_BINARY_DIR_ABSOLUTE} ${NETGEN_LIBRARY_DIR_ABSOLUTE})
+
+if (USE_SUPERBUILD)
   # execute the superbuild (this script will be invoked again without the
   # USE_SUPERBUILD option this time)
   include (cmake/SuperBuild.cmake)
   return()                      # stop processing this file further
-else()
-  project(Netgen)
-  if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-    set(CMAKE_INSTALL_PREFIX "${INSTALL_DIR_DEFAULT}" CACHE PATH "Install directory" FORCE)
-  endif()
 endif()
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -87,6 +164,7 @@ if(USE_CCACHE)
   find_program(CCACHE_FOUND NAMES ccache ccache.bat)
   if(CCACHE_FOUND)
     set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+    message(STATUS "Using ccache ${CCACHE_FOUND}")
   endif(CCACHE_FOUND)
 endif(USE_CCACHE)
 
@@ -115,61 +193,6 @@ include_directories ("${PROJECT_BINARY_DIR}")
 
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 
-if(USE_PYTHON)
-    find_package(PythonInterp 3 REQUIRED)
-    find_package(PythonLibs 3 REQUIRED)
-
-    execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1,0,''))" OUTPUT_VARIABLE PYTHON_PACKAGES_INSTALL_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
-    file(TO_CMAKE_PATH ${PYTHON_PACKAGES_INSTALL_DIR} PYTHON_PACKAGES_INSTALL_DIR)
-endif(USE_PYTHON)
-
-set(NG_INSTALL_SUFFIX netgen CACHE STRING "Suffix appended to install directories (project name)")
-
-if(APPLE)
-  set(NG_INSTALL_DIR_BIN_DEFAULT Contents/MacOS)
-  set(NG_INSTALL_DIR_LIB_DEFAULT Contents/MacOS)
-  set(NG_INSTALL_DIR_CMAKE_DEFAULT Contents/Resources/CMake)
-  set(NG_INSTALL_DIR_PYTHON_DEFAULT Contents/Resources/${PYTHON_PACKAGES_INSTALL_DIR})
-  set(NG_INSTALL_DIR_RES_DEFAULT Contents/Resources/share)
-  set(NG_INSTALL_DIR_INCLUDE_DEFAULT Contents/Resources/include)
-
-  set(NG_RPATH_TOKEN "@loader_path")
-else(APPLE)
-  set(NG_INSTALL_DIR_BIN_DEFAULT bin)
-  set(NG_INSTALL_DIR_LIB_DEFAULT lib)
-  if(WIN32)
-    set(NG_INSTALL_DIR_CMAKE_DEFAULT cmake)
-  else(WIN32)
-    set(NG_INSTALL_DIR_CMAKE_DEFAULT lib/cmake/${NG_INSTALL_SUFFIX})
-  endif(WIN32)
-  set(NG_INSTALL_DIR_PYTHON_DEFAULT ${PYTHON_PACKAGES_INSTALL_DIR})
-  set(NG_INSTALL_DIR_RES_DEFAULT share)
-  set(NG_INSTALL_DIR_INCLUDE_DEFAULT include)
-
-  set(NG_RPATH_TOKEN "\$ORIGIN")
-endif(APPLE)
-
-set(NG_INSTALL_DIR_PYTHON ${NG_INSTALL_DIR_PYTHON_DEFAULT} CACHE STRING "Install directory for Python files")
-set(NG_INSTALL_DIR_BIN ${NG_INSTALL_DIR_BIN_DEFAULT} CACHE STRING "Install directory for executables")
-set(NG_INSTALL_DIR_LIB ${NG_INSTALL_DIR_LIB_DEFAULT} CACHE STRING "Install directory for libraries")
-set(NG_INSTALL_DIR_INCLUDE ${NG_INSTALL_DIR_INCLUDE_DEFAULT} CACHE STRING "Install directory for header files")
-set(NG_INSTALL_DIR_CMAKE ${NG_INSTALL_DIR_CMAKE_DEFAULT} CACHE STRING "Install directory for CMake files")
-set(NG_INSTALL_DIR_RES ${NG_INSTALL_DIR_RES_DEFAULT} CACHE STRING "Install directory for resources")
-
-get_filename_component(NETGEN_CMAKE_DIR_ABSOLUTE ${NG_INSTALL_DIR_CMAKE} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
-get_filename_component(NETGEN_BINARY_DIR_ABSOLUTE ${NG_INSTALL_DIR_BIN} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
-get_filename_component(NETGEN_LIBRARY_DIR_ABSOLUTE ${NG_INSTALL_DIR_LIB} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
-get_filename_component(NETGEN_INCLUDE_DIR_ABSOLUTE ${NG_INSTALL_DIR_INCLUDE} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
-get_filename_component(NETGEN_RESOURCE_DIR_ABSOLUTE ${NG_INSTALL_DIR_RES} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
-
-file(RELATIVE_PATH NETGEN_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${CMAKE_INSTALL_PREFIX})
-file(RELATIVE_PATH NETGEN_BINARY_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_BINARY_DIR_ABSOLUTE})
-file(RELATIVE_PATH NETGEN_LIBRARY_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_LIBRARY_DIR_ABSOLUTE})
-file(RELATIVE_PATH NETGEN_INCLUDE_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_INCLUDE_DIR_ABSOLUTE})
-file(RELATIVE_PATH NETGEN_RESOURCE_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_RESOURCE_DIR_ABSOLUTE})
-
-file(RELATIVE_PATH NETGEN_RPATH ${NETGEN_BINARY_DIR_ABSOLUTE} ${NETGEN_LIBRARY_DIR_ABSOLUTE})
-
 if(USE_PYTHON)
     get_filename_component(NETGEN_PYTHON_DIR_ABSOLUTE ${NG_INSTALL_DIR_PYTHON} ABSOLUTE BASE_DIR ${CMAKE_INSTALL_PREFIX})
     file(RELATIVE_PATH NETGEN_PYTHON_DIR ${NETGEN_CMAKE_DIR_ABSOLUTE} ${NETGEN_PYTHON_DIR_ABSOLUTE})
@@ -185,6 +208,10 @@ install(EXPORT netgen-targets DESTINATION ${NG_INSTALL_DIR_CMAKE} COMPONENT netg
 
 set(CMAKE_MACOSX_RPATH TRUE)
 set(CMAKE_INSTALL_RPATH "${NG_RPATH_TOKEN};${NG_RPATH_TOKEN}/${NETGEN_RPATH}")
+if(BUILD_FOR_CONDA)
+    file(RELATIVE_PATH py_rpath "/bin" "/${NG_INSTALL_DIR_LIB}")
+    set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH};${py_rpath}")
+endif(BUILD_FOR_CONDA)
 
 include (CheckIncludeFiles)
 check_include_files (dlfcn.h HAVE_DLFCN_H)
@@ -210,50 +237,75 @@ macro(get_dll_from_lib dll_path lib_path)
         get_filename_component(lib_name ${lib} name)
 endmacro()
 
+set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 set(CMAKE_CXX_STANDARD 17)
 if(WIN32)
     set(CMAKE_MFC_FLAG 0)
-    # build convenience (aka object) libraries in windows)
-    set(NG_LIB_TYPE OBJECT)
-else(WIN32)
-    # build shared libraries
-    set(NG_LIB_TYPE SHARED)
 endif(WIN32)
 if(APPLE)
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup")
 endif(APPLE)
 
+#######################################################################
+add_library(nglib ${NGLIB_LIBRARY_TYPE})
+if(USE_GUI)
+    add_library(nggui ${NGGUI_LIBRARY_TYPE})
+    if(WIN32)
+        set_target_properties( nggui PROPERTIES OUTPUT_NAME "libnggui")
+    endif(WIN32)
+endif(USE_GUI)
+
 #######################################################################
 if(NOT ZLIB_INCLUDE_DIRS)
   find_package(ZLIB REQUIRED)
 endif(NOT ZLIB_INCLUDE_DIRS)
-include_directories(${ZLIB_INCLUDE_DIRS})
+target_include_directories(nglib PRIVATE ${ZLIB_INCLUDE_DIRS})
+if(USE_GUI)
+    target_include_directories(nggui PRIVATE ${ZLIB_INCLUDE_DIRS})
+endif(USE_GUI)
 
 #######################################################################
+if(WIN32)
+    add_library(netgen_gui INTERFACE IMPORTED)
+else()
+    add_library(netgen_gui INTERFACE)
+endif()
+
 if (USE_GUI)
     find_package(TCL 8.5 REQUIRED)
+    find_package(TclStub 8.5 REQUIRED)
     find_package(Threads REQUIRED)
     if(APPLE)
           set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework AppKit")
+          target_link_libraries(netgen_gui INTERFACE "-framework AppKit")
     else(APPLE)
           find_package(X11 REQUIRED) 
+          target_link_libraries( netgen_gui INTERFACE ${X11_Xmu_LIB} ${X11_X11_LIB})
     endif(APPLE)
     find_package(OpenGL REQUIRED)
 
-    add_definitions(-DTCL -DOPENGL -DUSE_TOGL_2)
-    include_directories(${TCL_INCLUDE_PATH})
-    include_directories(${TK_INCLUDE_PATH})
-    set(LIBTOGL togl)
+    target_compile_definitions(netgen_gui INTERFACE -DTCL -DOPENGL -DUSE_TOGL_2 -DUSE_TCL_STUBS -DUSE_TK_STUBS)
+    target_include_directories(netgen_gui INTERFACE ${TCL_INCLUDE_PATH} ${TK_INCLUDE_PATH})
+    target_link_libraries(netgen_gui INTERFACE ${TCL_STUB_LIBRARY} ${TK_STUB_LIBRARY})
+    if(NOT EXISTS ${TK_INCLUDE_PATH}/tkWin.h AND EXISTS ${TK_INCLUDE_PATH}/../win/tkWin.h)
+       target_include_directories(netgen_gui INTERFACE ${TK_INCLUDE_PATH}/../win)
+    endif()
+    if(NOT EXISTS ${TK_INCLUDE_PATH}/x11/Xlib.h AND EXISTS ${TK_INCLUDE_PATH}/../xlib/X11/Xlib.h)
+       target_include_directories(netgen_gui INTERFACE ${TK_INCLUDE_PATH}/../xlib)
+    endif()
+
+    target_link_libraries(nggui PUBLIC nglib togl PRIVATE "$<BUILD_INTERFACE:netgen_python>" )
 
     if(WIN32)
-        add_definitions(-DTOGL_WGL)
-    else(WIN32)
-        if(APPLE)
-                ADD_DEFINITIONS(-DTOGL_NSOPENGL)
-        else(APPLE)
-                ADD_DEFINITIONS(-DTOGL_X11)
-        endif(APPLE)
-    endif(WIN32)
+      target_compile_definitions(netgen_gui INTERFACE -DTOGL_WGL)
+    endif()
+    if(APPLE)
+      target_compile_definitions(netgen_gui INTERFACE -DTOGL_NSOPENGL)
+    endif()
+    if(UNIX AND NOT APPLE)
+      target_compile_definitions(netgen_gui INTERFACE -DTOGL_X11)
+    endif()
+
 endif (USE_GUI)
 
 #######################################################################
@@ -264,47 +316,115 @@ else()
 endif()
 
 if (USE_PYTHON)
-    add_subdirectory(external_dependencies/pybind11)
-    find_path(PYBIND_INCLUDE_DIR pybind11/pybind11.h HINTS ${PYTHON_INCLUDE_DIR})
-    if( PYBIND_INCLUDE_DIR )
-        message(STATUS "Found Pybind11: ${PYBIND_INCLUDE_DIR}")
-    else( PYBIND_INCLUDE_DIR )
-        message(FATAL_ERROR "Could NOT find pybind11!")
-    endif( PYBIND_INCLUDE_DIR )
+    if (PREFER_SYSTEM_PYBIND11)
+      set(NG_INSTALL_PYBIND OFF)
+      find_package(pybind11 CONFIG REQUIRED)
+    else()
+      add_subdirectory(external_dependencies/pybind11)
+    endif()
 
-    target_include_directories(netgen_python INTERFACE ${PYBIND_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS})
-    if(NOT ${BUILD_FOR_CONDA})
-        # Don't link python libraries in conda environments
-        target_link_libraries(netgen_python INTERFACE ${PYTHON_LIBRARIES})
+    target_compile_definitions(netgen_python INTERFACE NG_PYTHON NETGEN_PYTHON)
+    target_include_directories(netgen_python INTERFACE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS})
+    target_include_directories(nglib PRIVATE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS})
+    if(Python3_LIBRARIES AND (WIN32 OR NOT BUILD_FOR_CONDA))
+        target_link_libraries(netgen_python INTERFACE ${Python3_LIBRARIES})
     endif()
 
     if(NG_INSTALL_PYBIND)
-      install(DIRECTORY ${PYBIND_INCLUDE_DIR}/pybind11 DESTINATION ${NG_INSTALL_DIR_INCLUDE} COMPONENT netgen_devel)
-      install(FILES ${PYBIND_INCLUDE_DIR}/../LICENSE DESTINATION ${NG_INSTALL_DIR_INCLUDE}/pybind11 COMPONENT netgen_devel)
+      install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${NG_INSTALL_DIR_INCLUDE} COMPONENT netgen_devel)
+      install(FILES ${pybind11_INCLUDE_DIR}/../LICENSE DESTINATION ${NG_INSTALL_DIR_INCLUDE}/pybind11 COMPONENT netgen_devel)
     endif(NG_INSTALL_PYBIND)
 endif (USE_PYTHON)
 
 #######################################################################
-add_library(netgen_mpi INTERFACE)
 add_library(netgen_metis INTERFACE)
 if (USE_MPI)
-    find_package(MPI REQUIRED)
-    target_include_directories(netgen_mpi INTERFACE ${MPI_CXX_INCLUDE_PATH})
-    target_link_libraries(netgen_mpi INTERFACE ${MPI_mpi_LIBRARY} ${MPI_CXX_LIBRARIES} )
-    target_compile_definitions(netgen_mpi INTERFACE PARALLEL )
+    set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
+    find_package(MPI)
 
     find_package(METIS REQUIRED)
     target_include_directories(netgen_metis INTERFACE ${METIS_INCLUDE_DIR})
     target_link_libraries(netgen_metis INTERFACE ${METIS_LIBRARY} )
     target_compile_definitions(netgen_metis INTERFACE METIS )
 endif (USE_MPI)
-install(TARGETS netgen_mpi netgen_metis ${NG_INSTALL_DIR})
 
 #######################################################################
+add_library(occ_libs INTERFACE IMPORTED)
 if (USE_OCC)
-    find_package(OpenCasCade REQUIRED)
-    add_definitions(-DOCCGEOMETRY -D_OCC64)
-    include_directories(${OCC_INCLUDE_DIR})
+    find_package(OpenCascade NAMES OpenCASCADE opencascade REQUIRED CMAKE_FIND_ROOT_PATH_BOTH)
+    add_definitions(-DOCCGEOMETRY)
+    set(OCC_LIBRARIES
+      TKBO
+      TKBRep
+      TKBool
+      TKCAF
+      TKCDF
+      TKFillet
+      TKG2d
+      TKG3d
+      TKGeomAlgo
+      TKGeomBase
+      TKHLR
+      TKLCAF
+      TKMath
+      TKMesh
+      TKOffset
+      TKPrim
+      TKService
+      TKShHealing
+      TKTopAlgo
+      TKV3d
+      TKVCAF
+      TKXCAF
+      TKXSBase
+      TKernel
+    )
+    if(${OpenCASCADE_MAJOR_VERSION}.${OpenCASCADE_MINOR_VERSION} VERSION_GREATER_EQUAL 7.8)
+      list(APPEND OCC_LIBRARIES TKDEIGES TKDESTEP TKDESTL)
+    else()
+      list(APPEND OCC_LIBRARIES
+        TKIGES
+        TKSTEP
+        TKSTL
+        TKXDEIGES
+        TKXDESTEP
+        TKSTEP209
+        TKSTEPAttr
+        TKSTEPBase
+        )
+    endif()
+    if(UNIX AND NOT APPLE)
+      list(PREPEND OCC_LIBRARIES -Wl,--start-group)
+      list(APPEND OCC_LIBRARIES -Wl,--end-group)
+    endif()
+
+    target_link_libraries(occ_libs INTERFACE ${OCC_LIBRARIES})
+    get_target_property(occ_include_dir TKernel INTERFACE_INCLUDE_DIRECTORIES)
+    if(NOT occ_include_dir)
+      set(occ_include_dir ${OpenCASCADE_INCLUDE_DIR})
+    endif()
+    target_include_directories(occ_libs INTERFACE ${occ_include_dir})
+    message(STATUS "OpenCasCade include dirs: ${occ_include_dir}")
+    if(NOT OpenCASCADE_BUILD_SHARED_LIBS)
+      if(OpenCASCADE_WITH_FREETYPE)
+        find_library( FREETYPE NAMES freetype HINTS ${OpenCASCADE_LIBRARY_DIR})
+        list(APPEND OCC_LIBRARIES ${FREETYPE})
+        target_link_libraries(occ_libs INTERFACE ${FREETYPE})
+        if(UNIX AND NOT APPLE)
+          find_package(Fontconfig REQUIRED)
+          target_link_libraries(occ_libs INTERFACE ${Fontconfig_LIBRARIES})
+        endif()
+      endif(OpenCASCADE_WITH_FREETYPE)
+      set(THREADS_PREFER_PTHREAD_FLAG ON)
+      find_package(Threads REQUIRED)
+      target_link_libraries(occ_libs INTERFACE Threads::Threads)
+    endif()
+    if(WIN32 AND USE_GUI)
+      target_link_libraries(nggui PRIVATE Ws2_32.lib)
+    endif(WIN32 AND USE_GUI)
+    if(USE_GUI)
+      target_link_libraries(nggui PRIVATE occ_libs)
+    endif(USE_GUI)
 endif (USE_OCC)
 
 #######################################################################
@@ -404,16 +524,18 @@ if(USE_CGNS)
     endif(NOT WIN32 AND NOT APPLE)
 endif(USE_CGNS)
 
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/netgen_version.hpp ${CMAKE_CURRENT_BINARY_DIR}/netgen_config.hpp DESTINATION ${NG_INSTALL_DIR_INCLUDE}/include COMPONENT netgen_devel)
+
+# include instead of add_subdirectory to recognize the generated source files properly
+include(rules/CMakeLists.txt)
+
+add_subdirectory(windows)
 add_subdirectory(libsrc)
 add_subdirectory(ng)
 add_subdirectory(tutorials)
 add_subdirectory(py_tutorials)
 add_subdirectory(doc)
-add_subdirectory(windows)
 add_subdirectory(nglib)
-if (USE_PYTHON)
-  add_subdirectory(python)
-endif (USE_PYTHON)
 add_subdirectory(tests)
 
 #######################################################################
@@ -460,6 +582,8 @@ if(USE_NATIVE_ARCH)
         else()
             message(STATUS "Build for generic CPU")
         endif()
+    elseif(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
+	# no flag necessary/available on Apple M1
     else()
         target_compile_options(ngcore PUBLIC "-march=native")
     endif(WIN32)
@@ -469,6 +593,9 @@ if(USE_NATIVE_ARCH)
     endif(APPLE)
 endif(USE_NATIVE_ARCH)
 
+if (USE_PYTHON)
+  add_subdirectory(python)
+endif (USE_PYTHON)
 
 #######################################################################
 # Debian packager
@@ -506,7 +633,17 @@ if(UNIX)
     endif(temp)
 endif(UNIX)
 
-if(APPLE)
+if(USE_PYTHON AND NOT SKBUILD)
+  # install egg file to let python/pip know that Netgen ist installed
+  file( WRITE "netgen_mesher-py3.egg-info"
+"Metadata-Version: 2.1
+Name: netgen-mesher
+Version: ${NETGEN_VERSION_MAJOR}.${NETGEN_VERSION_MINOR}.${NETGEN_VERSION_PATCH}.post${NETGEN_VERSION_TWEAK}
+")
+  install(FILES netgen_mesher-py3.egg-info DESTINATION ${NG_INSTALL_DIR_PYTHON} COMPONENT netgen)
+endif()
+
+if(APPLE AND NOT SKBUILD)
     # create some auxiliary files
     set(mac_startup ${CMAKE_CURRENT_BINARY_DIR}/startup.sh)
     file(WRITE ${mac_startup} "\
@@ -551,7 +688,7 @@ open -a /Applications/Utilities/Terminal.app $Netgen_MACOS/startup.sh || open -a
     install(FILES ${mac_plist} DESTINATION ${NG_INSTALL_DIR_BIN}/../)
     install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/netgen.icns DESTINATION ${NG_INSTALL_DIR_RES}/../ RENAME Netgen.icns)
 
-endif(APPLE)
+endif(APPLE AND NOT SKBUILD)
 
 if(NOT APPLE)
   include(CPack) 

CONTRIBUTING.md

@@ -0,0 +1,18 @@
+
+# How to Contribute
+
+## Reporting issues
+
+If you have a problem using Netgen/NGSolve consider asking a question in our [forum](https://ngsolve.org/forum).
+
+If you found a bug create an issue in the [Github Issue Tracker](https://github.com/NGSolve/netgen/issues). Please be as specific as possible, issues with a reproducible minimal failing example will get more attention than unspecific one liners :)
+
+## Contributing patches
+
+We love and want to encourage community engagement and will review and accept patches and contributions to this project. There are just a few steps to follow:
+
+On your first contribution, to clear any legal questions, we ask you to sign our [Contributor License Agreement](CLA.pdf). Generally you have to sign this only once for Netgen or NGSolve. Please send the signed agreement to <joachim.schoeberl@tuwien.ac.at>.
+
+Place a pull request on GitHub. From there we will pull it into our internal testing environment and, if approved, merge it into the main codebase.
+
+If you have any questions feel free to ask on the [forum](https://ngsolve.org/forum).

README.md

@@ -1,3 +1,7 @@
 Netgen mesh generator
 
-NETGEN is an automatic 3d tetrahedral mesh generator. It accepts input from constructive solid geometry (CSG) or boundary representation (BRep) from STL file format. The connection to a geometry kernel allows the handling of IGES and STEP files. NETGEN contains modules for mesh optimization and hierarchical mesh refinement. Netgen 6.x supports scripting via a Python interface. Netgen is open source based on the LGPL license. It is available for Unix/Linux, Windows, and OSX.
+NETGEN is an automatic 3d tetrahedral mesh generator. It accepts input from constructive solid geometry (CSG) or boundary representation (BRep) from STL file format. The connection to a geometry kernel allows the handling of IGES and STEP files. NETGEN contains modules for mesh optimization and hierarchical mesh refinement. Netgen 6.x supports scripting via a Python interface. Netgen is open source based on the LGPL license. It is available for Unix/Linux, Windows, and OSX.
+
+Find the Open Source Community on https://ngsolve.org
+Support & Services: https://cerbsim.com
+

cmake/NetgenConfig.cmake.in

@@ -32,19 +32,20 @@ set(NETGEN_MKL_LIBRARIES "@MKL_LIBRARIES@")
 set(NETGEN_MPI_CXX_INCLUDE_PATH "@MPI_CXX_INCLUDE_PATH@")
 set(NETGEN_MPI_CXX_LIBRARIES "@MPI_CXX_LIBRARIES@")
 set(NETGEN_NUMA_LIBRARY "@NUMA_LIBRARY@")
-set(NETGEN_OCC_INCLUDE_DIR "@OCC_INCLUDE_DIR@")
-set(NETGEN_OCC_LIBRARIES_BIN "@OCC_LIBRARIES_BIN@")
+set(NETGEN_OCC_DIR "@OpenCasCade_DIR@")
+set(NETGEN_OCC_INCLUDE_DIR "@OpenCASCADE_INCLUDE_DIR@")
+set(NETGEN_OCC_LIBRARIES_BIN "@OpenCASCADE_BINARY_DIR@")
 set(NETGEN_OCC_LIBRARIES "@OCC_LIBRARIES@")
-set(NETGEN_OCC_LIBRARY_DIR "@OCC_LIBRARY_DIR@")
+set(NETGEN_OCC_LIBRARY_DIR "@OpenCASCADE_LIBRARY_DIR@")
 set(NETGEN_OPENGL_LIBRARIES "@OPENGL_LIBRARIES@")
-set(NETGEN_PYTHON_EXECUTABLE "@PYTHON_EXECUTABLE@")
-set(NETGEN_PYTHON_INCLUDE_DIRS "@PYTHON_INCLUDE_DIRS@")
-set(NETGEN_PYTHON_LIBRARIES "@PYTHON_LIBRARIES@")
+set(NETGEN_PYTHON_EXECUTABLE "@Python3_EXECUTABLE@")
+set(NETGEN_PYTHON_INCLUDE_DIRS "@Python3_INCLUDE_DIRS@")
+set(NETGEN_PYTHON_LIBRARIES "@Python3_LIBRARIES@")
 set(NETGEN_TCL_INCLUDE_PATH "@TCL_INCLUDE_PATH@")
-set(NETGEN_TCL_LIBRARY "@TCL_LIBRARY@")
+set(NETGEN_TCL_LIBRARY "@TCL_STUB_LIBRARY@")
 set(NETGEN_TK_DND_LIBRARY "@TK_DND_LIBRARY@")
 set(NETGEN_TK_INCLUDE_PATH "@TK_INCLUDE_PATH@")
-set(NETGEN_TK_LIBRARY "@TK_LIBRARY@")
+set(NETGEN_TK_LIBRARY "@TK_STUB_LIBRARY@")
 set(NETGEN_X11_X11_LIB "@X11_X11_LIB@")
 set(NETGEN_X11_Xmu_LIB "@X11_Xmu_LIB@")
 set(NETGEN_ZLIB_INCLUDE_DIRS "@ZLIB_INCLUDE_DIRS@")

cmake/SuperBuild.cmake

@@ -1,27 +1,80 @@
 include (ExternalProject)
 
+option( BUILD_ZLIB "Build and link static version of zlib (useful for pip binaries)" OFF )
+option( BUILD_OCC "Build and link static version of occ (useful for pip binaries)" OFF )
 set_property (DIRECTORY PROPERTY EP_PREFIX dependencies)
 
 set (NETGEN_DEPENDENCIES)
 set (LAPACK_DEPENDENCIES)
 set (NETGEN_CMAKE_ARGS "" CACHE INTERNAL "")
+set (SUBPROJECT_CMAKE_ARGS "" CACHE INTERNAL "")
+
+set (SUBPROJECT_ARGS
+    LIST_SEPARATOR |
+    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dependencies
+)
+
+if (EMSCRIPTEN)
+    set (SUBPROJECT_ARGS
+        ${SUBPROJECT_ARGS}
+        CMAKE_COMMAND emcmake ${CMAKE_COMMAND})
+endif()
+
+# only show output on failure in ci-builds
+if(DEFINED ENV{CI})
+    set (SUBPROJECT_ARGS
+        ${SUBPROJECT_ARGS}
+        LOG_DOWNLOAD ON
+        LOG_BUILD ON
+        LOG_INSTALL ON
+        LOG_CONFIGURE ON
+    )
+    if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0")
+        set (SUBPROJECT_ARGS
+            ${SUBPROJECT_ARGS}
+            LOG_OUTPUT_ON_FAILURE ON
+            LOG_MERGED_STDOUTERR ON
+        )
+    endif()
+endif()
+
+
+set (NETGEN_CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} )
 
 macro(set_vars VAR_OUT)
   foreach(varname ${ARGN})
     if(NOT "${${varname}}" STREQUAL "")
-      string(REPLACE ";" "$<SEMICOLON>" varvalue "${${varname}}" )
-      set(${VAR_OUT} ${${VAR_OUT}};-D${varname}=${varvalue} CACHE INTERNAL "")
+      string(REPLACE ";" "|" varvalue "${${varname}}" )
+      set(${VAR_OUT} "${${VAR_OUT}};-D${varname}=${varvalue}" CACHE INTERNAL "")
     endif()
   endforeach()
 endmacro()
+#######################################################################
+
+set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_OSX_DEPLOYMENT_TARGET)
+set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_OSX_SYSROOT)
+set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_C_COMPILER)
+set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_CXX_COMPILER)
+set_vars(SUBPROJECT_CMAKE_ARGS CMAKE_BUILD_TYPE)
+
+set(SUBPROJECT_CMAKE_ARGS "${SUBPROJECT_CMAKE_ARGS};-DCMAKE_POSITION_INDEPENDENT_CODE=ON;-DCMAKE_POLICY_VERSION_MINIMUM=3.5" CACHE INTERNAL "")
+
+if(USE_CCACHE)
+  find_program(CCACHE_FOUND NAMES ccache ccache.bat)
+  if(CCACHE_FOUND)
+      set(SUBPROJECT_CMAKE_ARGS "${SUBPROJECT_CMAKE_ARGS};-DCMAKE_CXX_COMPILER_LAUNCHER=${CCACHE_FOUND}" CACHE INTERNAL "")
+  endif()
+endif()
+
 #######################################################################
 set (DEPS_DOWNLOAD_URL "https://github.com/NGSolve/ngsolve_dependencies/releases/download/v1.0.0" CACHE STRING INTERNAL)
-set (OCC_DOWNLOAD_URL_WIN "${DEPS_DOWNLOAD_URL}/occ_win64.zip" CACHE STRING INTERNAL)
+set (OCC_DOWNLOAD_URL_WIN "${DEPS_DOWNLOAD_URL}/occ75_win64.zip" CACHE STRING INTERNAL)
 set (TCLTK_DOWNLOAD_URL_WIN "${DEPS_DOWNLOAD_URL}/tcltk_win64.zip" CACHE STRING INTERNAL)
 set (ZLIB_DOWNLOAD_URL_WIN "${DEPS_DOWNLOAD_URL}/zlib_win64.zip" CACHE STRING INTERNAL)
 set (CGNS_DOWNLOAD_URL_WIN "${DEPS_DOWNLOAD_URL}/cgns_win64.zip" CACHE STRING INTERNAL)
 set (CGNS_DOWNLOAD_URL_MAC "${DEPS_DOWNLOAD_URL}/cgns_mac.zip" CACHE STRING INTERNAL)
 
+
 if(UNIX)
   message("Checking for write permissions in install directory...")
   execute_process(COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX})
@@ -31,10 +84,82 @@ if(UNIX)
   endif()
 endif(UNIX)
 
-if(NOT WIN32)
-    find_package(ZLIB REQUIRED)
-    set_vars(NETGEN_CMAKE_ARGS ZLIB_INCLUDE_DIRS ZLIB_LIBRARIES)
-endif(NOT WIN32)
+if(USE_OCC)
+if(BUILD_OCC)
+  set(OCC_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/occ)
+
+  ExternalProject_Add(project_occ
+    # URL https://github.com/Open-Cascade-SAS/OCCT/archive/refs/tags/V7_6_3.zip
+    # URL_MD5 2426e373903faabbd4f96a01a934b66d
+    # URL https://github.com/Open-Cascade-SAS/OCCT/archive/refs/tags/V7_7_2.zip
+    # URL_MD5 533eb4f18af0f77ae321b158caeaee79
+    URL https://github.com/Open-Cascade-SAS/OCCT/archive/refs/tags/V7_8_1.zip
+    URL_MD5 bf62952a03696dab9e4272aa8efacb1a
+    DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies
+    ${SUBPROJECT_ARGS}
+    CMAKE_ARGS
+         -DCMAKE_INSTALL_PREFIX=${OCC_DIR}
+         -DCMAKE_PREFIX_PATH=${OCC_DIR}
+         -DBUILD_LIBRARY_TYPE:STRING=Static
+         -DBUILD_MODULE_FoundationClasses:BOOL=ON
+         -DBUILD_MODULE_ModelingData:BOOL=ON
+         -DBUILD_MODULE_ModelingAlgorithms:BOOL=ON
+         -DBUILD_MODULE_DataExchange:BOOL=ON
+         -DBUILD_MODULE_Visualization:BOOL=OFF
+         -DBUILD_MODULE_ApplicationFramework:BOOL=OFF
+         -DBUILD_MODULE_Draw:BOOL=OFF
+         -DBUILD_MODULE_DETools:BOOL=OFF
+         -DUSE_FREETYPE:BOOL=OFF
+         -DUSE_OPENGL:BOOL=OFF
+         -DUSE_XLIB:BOOL=OFF
+         -DBUILD_DOC_Overview:BOOL=OFF
+         ${SUBPROJECT_CMAKE_ARGS}
+    UPDATE_COMMAND ""
+    )
+
+  list(APPEND NETGEN_DEPENDENCIES project_occ)
+  set(OpenCascade_ROOT ${OCC_DIR})
+else(BUILD_OCC)
+    find_package(OpenCascade NAMES OpenCasCade OpenCASCADE opencascade)
+    if(NOT OpenCascade_FOUND)
+      message(FATAL_ERROR "Opencascade not found, either\n\
+      - install pip packages netgen-occt-devel netgen-occ\n\
+      - set OpenCascade_DIR to a directory containting opencascadeConfig.cmake\n\
+      - build OpenCascade automatically by passing -DBUILD_OCC=ON\n\
+      - disable OpenCascade by passing -DUSE_OCC=OFF\n\
+      ")
+    endif()
+endif(BUILD_OCC)
+endif(USE_OCC)
+
+if(BUILD_ZLIB)
+  set(ZLIB_ROOT ${CMAKE_CURRENT_BINARY_DIR}/dependencies/zlib)
+  ExternalProject_Add(project_zlib
+    ${SUBPROJECT_ARGS}
+    URL https://github.com/madler/zlib/archive/refs/tags/v1.2.11.zip
+    URL_MD5 9d6a627693163bbbf3f26403a3a0b0b1
+    DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies
+    CMAKE_ARGS
+         -DCMAKE_INSTALL_PREFIX=${ZLIB_ROOT}
+         ${SUBPROJECT_CMAKE_ARGS}
+    UPDATE_COMMAND "" # Disable update
+    BUILD_IN_SOURCE 1
+    )
+
+  list(APPEND NETGEN_DEPENDENCIES project_zlib)
+  if(WIN32)
+    # force linking the static library
+    set(ZLIB_INCLUDE_DIRS ${ZLIB_ROOT}/include)
+    set(ZLIB_LIBRARIES ${ZLIB_ROOT}/lib/zlibstatic.lib)
+    set(ZLIB_LIBRARY_RELEASE ${ZLIB_ROOT}/lib/zlibstatic.lib)
+  else(WIN32)
+    set(ZLIB_INCLUDE_DIRS ${ZLIB_ROOT}/include)
+    set(ZLIB_LIBRARIES ${ZLIB_ROOT}/lib/libz.a)
+    set(ZLIB_LIBRARY_RELEASE ${ZLIB_ROOT}/lib/libz.a)
+  endif(WIN32)
+else()
+    include(cmake/external_projects/zlib.cmake)
+endif()
 
 #######################################################################
 if (USE_PYTHON)
@@ -50,14 +175,20 @@ if (USE_PYTHON)
     else( PYBIND_INCLUDE_DIR )
         message(FATAL_ERROR "Could NOT find pybind11!")
     endif( PYBIND_INCLUDE_DIR )
-    find_package(PythonInterp 3 REQUIRED)
-    find_package(PythonLibs 3 REQUIRED)
+    if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.18)
+      find_package(Python3 COMPONENTS Interpreter Development.Module)
+      if(NOT EMSCRIPTEN)
+          find_package(Python3 COMPONENTS Interpreter Development.Embed)
+      endif()
+    else()
+      find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
+    endif()
 
     set_vars(NETGEN_CMAKE_ARGS
-      PYTHON_INCLUDE_DIRS
-      PYTHON_LIBRARIES
-      PYTHON_EXECUTABLE
-      PYTHON_VERSION
+      Python3_INCLUDE_DIRS
+      Python3_LIBRARIES
+      Python3_EXECUTABLE
+      Python3_VERSION
       PYBIND_INCLUDE_DIR
       NG_INSTALL_PYBIND
       )
@@ -65,23 +196,6 @@ endif (USE_PYTHON)
 
 #######################################################################
 
-if(USE_OCC AND WIN32 AND NOT OCC_INCLUDE_DIR)
-    ExternalProject_Add(win_download_occ
-      PREFIX ${CMAKE_CURRENT_BINARY_DIR}/tcl
-      URL ${OCC_DOWNLOAD_URL_WIN}
-      UPDATE_COMMAND "" # Disable update
-      BUILD_IN_SOURCE 1
-      CONFIGURE_COMMAND ""
-      BUILD_COMMAND ""
-      INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory . ${CMAKE_INSTALL_PREFIX}
-      LOG_DOWNLOAD 1
-      )
-    list(APPEND NETGEN_DEPENDENCIES win_download_occ)
-endif(USE_OCC AND WIN32 AND NOT OCC_INCLUDE_DIR)
-
-#######################################################################
-
-include(cmake/external_projects/zlib.cmake)
 if(USE_GUI)
   include(cmake/external_projects/tcltk.cmake)
 endif(USE_GUI)
@@ -92,7 +206,6 @@ endif(USE_CGNS)
 
 #######################################################################
 if(USE_MPI)
-  if(UNIX)
     if (METIS_DIR)
       message(STATUS "Using external METIS at: ${METIS_DIR}")
     else (METIS_DIR)
@@ -103,27 +216,24 @@ if(USE_MPI)
 	include(cmake/external_projects/metis.cmake)
       endif(NOT METIS_FOUND)
     endif(METIS_DIR)
-  else(UNIX)
-    find_package(METIS REQUIRED)
-  endif(UNIX)
 endif(USE_MPI)
 
 
 #######################################################################
 # propagate cmake variables to Netgen subproject
 set_vars( NETGEN_CMAKE_ARGS
-  CMAKE_CXX_COMPILER
-  CMAKE_BUILD_TYPE
+  CMAKE_MODULE_LINKER_FLAGS
+  CMAKE_MODULE_LINKER_FLAGS_RELEASE
   CMAKE_SHARED_LINKER_FLAGS
   CMAKE_SHARED_LINKER_FLAGS_RELEASE
   CMAKE_CXX_FLAGS
   CMAKE_CXX_FLAGS_RELEASE
-  CMAKE_OSX_DEPLOYMENT_TARGET
-  CMAKE_OSX_SYSROOT
+  CMAKE_STRIP
 
   USE_GUI
   USE_PYTHON
   USE_MPI
+  USE_MPI_WRAPPER
   USE_VT
   USE_VTUNE
   USE_NUMA
@@ -136,15 +246,33 @@ set_vars( NETGEN_CMAKE_ARGS
   USE_INTERNAL_TCL
   INSTALL_PROFILES
   INTEL_MIC
-  CMAKE_PREFIX_PATH
   CMAKE_INSTALL_PREFIX
   ENABLE_UNIT_TESTS
   ENABLE_CPP_CORE_GUIDELINES_CHECK
   USE_SPDLOG
   DEBUG_LOG
   CHECK_RANGE
+  TRACE_MEMORY
   BUILD_STUB_FILES
   BUILD_FOR_CONDA
+  NG_COMPILE_FLAGS
+  OpenCascade_ROOT
+  ZLIB_INCLUDE_DIRS
+  ZLIB_LIBRARIES
+  ZLIB_LIBRARY_RELEASE
+  ZLIB_ROOT
+
+  NGLIB_LIBRARY_TYPE
+  NGCORE_LIBRARY_TYPE
+  NGGUI_LIBRARY_TYPE
+
+  NG_INSTALL_DIR_PYTHON
+  NG_INSTALL_DIR_BIN
+  NG_INSTALL_DIR_LIB
+  NG_INSTALL_DIR_INCLUDE
+  NG_INSTALL_DIR_CMAKE
+  NG_INSTALL_DIR_RES
+  NG_INSTALL_SUFFIX
   )
 
 # propagate all variables set on the command line using cmake -DFOO=BAR
@@ -152,9 +280,10 @@ set_vars( NETGEN_CMAKE_ARGS
 get_cmake_property(CACHE_VARS CACHE_VARIABLES)
 foreach(CACHE_VAR ${CACHE_VARS})
   get_property(CACHE_VAR_HELPSTRING CACHE ${CACHE_VAR} PROPERTY HELPSTRING)
-  if(CACHE_VAR_HELPSTRING STREQUAL "No help, variable specified on the command line.")
+  if(CACHE_VAR_HELPSTRING STREQUAL "No help, variable specified on the command line." AND NOT CACHE_VAR STREQUAL "CMAKE_OSX_ARCHITECTURES")
     get_property(CACHE_VAR_TYPE CACHE ${CACHE_VAR} PROPERTY TYPE)
-    set(NETGEN_CMAKE_ARGS ${NETGEN_CMAKE_ARGS};-D${CACHE_VAR}:${CACHE_VAR_TYPE}=${${CACHE_VAR}} CACHE INTERNAL "")
+    string(REPLACE ";" "|" varvalue "${${CACHE_VAR}}" )
+    set(NETGEN_CMAKE_ARGS ${NETGEN_CMAKE_ARGS};-D${CACHE_VAR}:${CACHE_VAR_TYPE}=${varvalue} CACHE INTERNAL "")
   endif()
 endforeach()
 
@@ -164,10 +293,17 @@ else()
   set(NETGEN_BUILD_COMMAND ${CMAKE_COMMAND} --build ${CMAKE_CURRENT_BINARY_DIR}/netgen --config ${CMAKE_BUILD_TYPE})
 endif()
 
+
+string(REPLACE ";" "|" NETGEN_CMAKE_PREFIX_PATH_ALT_SEP "${NETGEN_CMAKE_PREFIX_PATH}")
 ExternalProject_Add (netgen
+  ${SUBPROJECT_ARGS}
   DEPENDS ${NETGEN_DEPENDENCIES}
   SOURCE_DIR ${PROJECT_SOURCE_DIR}
-  CMAKE_ARGS -DUSE_SUPERBUILD=OFF ${NETGEN_CMAKE_ARGS}
+  CMAKE_ARGS
+      -DUSE_SUPERBUILD=OFF
+      ${NETGEN_CMAKE_ARGS}
+      ${SUBPROJECT_CMAKE_ARGS}
+      -DCMAKE_PREFIX_PATH=${NETGEN_CMAKE_PREFIX_PATH_ALT_SEP}
   INSTALL_COMMAND ""
   BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/netgen
   BUILD_COMMAND ${NETGEN_BUILD_COMMAND}

cmake/cmake_modules/FindOpenCasCade.cmake

@@ -1,105 +0,0 @@
-# Try to find OCC
-# Once done this will define
-#
-# OCC_FOUND          - system has OCC - OpenCASCADE
-# OCC_INCLUDE_DIR    - where the OCC include directory can be found
-# OCC_LIBRARY_DIR    - where the OCC library directory can be found
-# OCC_LIBRARIES      - Link this to use OCC
-
-if(WIN32)
-    find_path(OCC_INCLUDE_DIR Standard_Version.hxx PATH_SUFFIXES inc ../inc)
-    find_library(OCC_LIBRARY TKernel)
-else(WIN32)
-    find_path(OCC_INCLUDE_DIR Standard_Version.hxx
-      /usr/include/opencascade
-      /usr/local/include/opencascade
-      /usr/include/oce
-      /usr/local/include/oce
-      /opt/opencascade/include
-      /opt/opencascade/inc
-    )
-    find_library(OCC_LIBRARY TKernel
-      /usr/lib
-      /usr/local/lib
-      /opt/opencascade/lib
-    )
-endif(WIN32)
-
-if(OCC_LIBRARY AND NOT OCC_LIBRARY_DIR)
-    get_filename_component(OCC_LIBRARY_DIR ${OCC_LIBRARY} PATH)
-endif(OCC_LIBRARY)
-
-if(OCC_INCLUDE_DIR)
-    file(STRINGS ${OCC_INCLUDE_DIR}/Standard_Version.hxx OCC_MAJOR
-      REGEX "#define OCC_VERSION_MAJOR.*"
-    )
-    string(REGEX MATCH "[0-9]+" OCC_MAJOR ${OCC_MAJOR})
-    file(STRINGS ${OCC_INCLUDE_DIR}/Standard_Version.hxx OCC_MINOR
-      REGEX "#define OCC_VERSION_MINOR.*"
-    )
-    string(REGEX MATCH "[0-9]+" OCC_MINOR ${OCC_MINOR})
-    file(STRINGS ${OCC_INCLUDE_DIR}/Standard_Version.hxx OCC_MAINT
-      REGEX "#define OCC_VERSION_MAINTENANCE.*"
-    )
-    string(REGEX MATCH "[0-9]+" OCC_MAINT ${OCC_MAINT})
-
-    set(OCC_VERSION_STRING "${OCC_MAJOR}.${OCC_MINOR}.${OCC_MAINT}")
-endif(OCC_INCLUDE_DIR)
-
-
-set(OCC_LIBRARY_NAMES
-    TKBO
-    TKBool
-    TKBRep
-    TKCAF
-    TKCDF
-    TKernel
-    TKG2d
-    TKG3d
-    TKGeomAlgo
-    TKGeomBase
-    TKHLR
-    TKIGES
-    TKLCAF
-    TKMath
-    TKMesh
-    TKOffset
-    TKPrim
-    TKService
-    TKShHealing
-    TKSTEP
-    TKSTEP209
-    TKSTEPAttr
-    TKSTEPBase
-    TKSTL
-    TKTopAlgo
-    TKV3d
-    TKXCAF
-    TKXDEIGES
-    TKXDESTEP
-    TKXSBase
-)
-
-if(OCC_LINK_FREETYPE)
-    set(OCC_LIBRARY_NAMES ${OCC_LIBRARY_NAMES} freetype)
-endif(OCC_LINK_FREETYPE)
-
-if(OCC_VERSION_STRING VERSION_GREATER_EQUAL "7.3.0")
-    set(OCC_LIBRARY_NAMES ${OCC_LIBRARY_NAMES} TKVCAF)
-endif()
-
-foreach( libname ${OCC_LIBRARY_NAMES} )
-    find_library( ${libname} ${libname} ${OCC_LIBRARY_DIR} NO_DEFAULT_PATH)
-    set(OCC_LIBRARIES ${OCC_LIBRARIES} ${${libname}})
-endforeach()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(OCC REQUIRED_VARS OCC_INCLUDE_DIR VERSION_VAR OCC_VERSION_STRING ${OCC_LIBRARIY_NAMES})
-
-if(OCC_FOUND)
-    message(STATUS "-- Found OpenCASCADE version: ${OCC_VERSION_STRING}")
-    message(STATUS "-- OpenCASCADE include directory: ${OCC_INCLUDE_DIR}")
-    message(STATUS "-- OpenCASCADE shared libraries directory: ${OCC_LIBRARY_DIR}")
-    message(STATUS "-- OpenCASCADE shared libraries :\n ${OCC_LIBRARIES}")
-endif(OCC_FOUND)
-

cmake/external_projects/catch.cmake

@@ -4,7 +4,7 @@ ExternalProject_Add(
     project_catch
     PREFIX ${CMAKE_BINARY_DIR}/catch
     GIT_REPOSITORY https://github.com/catchorg/Catch2.git
-    GIT_TAG v2.0.1
+    GIT_TAG v2.13.7
     TIMEOUT 10
     UPDATE_COMMAND "" # ${GIT_EXECUTABLE} pull
     CONFIGURE_COMMAND ""

cmake/external_projects/metis.cmake

@@ -3,8 +3,8 @@ set(METIS_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/metis)
 
 ExternalProject_Add(project_metis
   PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dependencies
-  URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p6.tar.gz
-  URL_MD5 55fc654bb838846b856ba898795143f1
+  URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p12.tar.gz
+  URL_MD5 6cd66f75f88dfa2cf043de011f85d8bc
   DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies
   CMAKE_ARGS
          -DGKLIB_PATH=${METIS_SRC_DIR}/GKlib

cmake/external_projects/tcltk.cmake

@@ -1,3 +1,73 @@
+if(UNIX AND NOT APPLE)
+  set (LINUX TRUE)
+endif()
+if(LINUX)
+    find_package(TclStub 8.5 REQUIRED)
+else(LINUX)
+if(SKBUILD)
+# we are building a pip package - download the tcl/tk sources matching the tkinter version (for private headers not shipped with python)
+
+execute_process(COMMAND ${Python3_EXECUTABLE} -c
+"import tkinter;print(tkinter.Tcl().eval('info patchlevel').replace('.','-'))"
+OUTPUT_VARIABLE PYTHON_TCL_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+set(TCL_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/src/project_tcl)
+set(TK_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/src/project_tk)
+
+ExternalProject_Add(project_tcl
+  URL "https://github.com/tcltk/tcl/archive/refs/tags/core-${PYTHON_TCL_VERSION}.zip"
+  UPDATE_COMMAND ""
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  INSTALL_COMMAND ""
+  ${SUBPROJECT_ARGS}
+  DOWNLOAD_DIR download_tcl
+)
+ExternalProject_Add(project_tk
+  URL "https://github.com/tcltk/tk/archive/refs/tags/core-${PYTHON_TCL_VERSION}.zip"
+  UPDATE_COMMAND ""
+  CONFIGURE_COMMAND ""
+  INSTALL_COMMAND ""
+  BUILD_COMMAND ${CMAKE_COMMAND} -E copy_directory macosx generic
+  ${SUBPROJECT_ARGS}
+  DOWNLOAD_DIR download_tk
+  BUILD_IN_SOURCE 1
+)
+
+set(TCL_INCLUDE_PATH ${TCL_DIR}/generic)
+set(TK_INCLUDE_PATH ${TK_DIR}/generic)
+list(APPEND NETGEN_DEPENDENCIES project_tcl project_tk)
+
+if(APPLE OR WIN32)
+    execute_process(COMMAND ${Python3_EXECUTABLE} -c "import sys; print(sys.prefix)" OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE)
+    file(TO_CMAKE_PATH ${PYTHON_PREFIX} PYTHON_PREFIX)
+
+    set(tcl_find_args
+        REQUIRED
+        NO_DEFAULT_PATH
+        NO_PACKAGE_ROOT_PATH
+        NO_CMAKE_PATH
+        NO_CMAKE_ENVIRONMENT_PATH
+        NO_SYSTEM_ENVIRONMENT_PATH
+        NO_CMAKE_SYSTEM_PATH
+        NO_CMAKE_FIND_ROOT_PATH
+        HINTS
+        ${PYTHON_PREFIX}/lib
+        ${PYTHON_PREFIX}/tcl
+        ${PYTHON_PREFIX}/Frameworks
+        ${PYTHON_PREFIX}/Frameworks/Tcl.framework
+        ${PYTHON_PREFIX}/Frameworks/Tk.framework
+        )
+    find_library(TCL_STUB_LIBRARY NAMES tclstub85 tclstub8.5 tclstub86 tclstub8.6 ${tcl_find_args})
+    find_library(TK_STUB_LIBRARY NAMES tkstub85 tkstub8.5 tkstub86 tkstub8.6 ${tcl_find_args})
+    find_library(TCL_LIBRARY NAMES tcl85 tcl8.5 tcl86 tcl8.6 tcl86t Tcl ${tcl_find_args})
+    find_library(TK_LIBRARY NAMES tk85 tk8.5 tk86 tk8.6 tk86t Tk ${tcl_find_args})
+else()
+    # use system tcl/tk on linux
+    find_package(TclStub REQUIRED)
+endif()
+
+else(SKBUILD)
 if(APPLE)
   set(tcl_prefix ${CMAKE_INSTALL_PREFIX})
   # URL "http://sourceforge.net/projects/tcl/files/Tcl/8.6.9/tcl8.6.9-src.tar.gz"
@@ -10,10 +80,7 @@ if(APPLE)
     CONFIGURE_COMMAND ../project_tcl/macosx/configure --enable-threads --enable-framework --prefix=${tcl_prefix} --libdir=${tcl_prefix}/Contents/Frameworks --bindir=${tcl_prefix}/Contents/Frameworks/Tcl.framework/bin
     BUILD_COMMAND make -j4 binaries libraries
     INSTALL_COMMAND make install-binaries install-headers install-libraries install-private-headers
-    LOG_DOWNLOAD 1
-    LOG_BUILD 1
-    LOG_CONFIGURE 1
-    LOG_INSTALL 1
+    ${SUBPROJECT_ARGS}
     )
 
   # URL "http://sourceforge.net/projects/tcl/files/Tcl/8.6.9/tk8.6.9.1-src.tar.gz"
@@ -27,14 +94,11 @@ if(APPLE)
     CONFIGURE_COMMAND ../project_tk/macosx/configure --enable-aqua=yes --enable-threads --enable-framework --prefix=${tcl_prefix} --libdir=${tcl_prefix}/Contents/Frameworks --bindir=${tcl_prefix}/Contents/Frameworks/Tcl.framework/bin --with-tcl=${tcl_prefix}/Contents/Frameworks/Tcl.framework
     BUILD_COMMAND make -j4 binaries libraries
     INSTALL_COMMAND make install-binaries install-headers install-libraries install-private-headers
-    LOG_DOWNLOAD 1
-    LOG_BUILD 1
-    LOG_CONFIGURE 1
-    LOG_INSTALL 1
+    ${SUBPROJECT_ARGS}
     )
 
   ExternalProject_Add(project_tkdnd
-    URL "http://sourceforge.net/projects/tkdnd/files/TkDND/TkDND%202.8/tkdnd2.8-src.tar.gz"
+    URL "https://src.fedoraproject.org/repo/pkgs/tkdnd/tkdnd2.8-src.tar.gz/a6d47a996ea957416469b12965d4db91/tkdnd2.8-src.tar.gz"
     URL_MD5 a6d47a996ea957416469b12965d4db91
     DEPENDS project_tcl project_tk
     DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies
@@ -45,10 +109,8 @@ if(APPLE)
            -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/Contents/MacOS
 	   -DTCL_INCLUDE_PATH=${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tcl.framework/Headers
 	   -DTK_INCLUDE_PATH=${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tk.framework/Headers
-    LOG_DOWNLOAD 1
-    LOG_CONFIGURE 1
-    LOG_BUILD 1
-    LOG_INSTALL 1
+     -DCMAKE_POLICY_VERSION_MINIMUM=3.5
+    ${SUBPROJECT_ARGS}
   )
 
   list(APPEND NETGEN_DEPENDENCIES project_tcl project_tk project_tkdnd)
@@ -58,6 +120,9 @@ if(APPLE)
   set(TK_LIBRARY ${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tk.framework)
   set(TK_INCLUDE_PATH ${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tk.framework/Headers)
 
+  set(TCL_STUB_LIBRARY ${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tcl.framework/libtclstub8.6.a)
+  set(TK_STUB_LIBRARY ${CMAKE_INSTALL_PREFIX}/Contents/Frameworks/Tk.framework/libtkstub8.6.a)
+
 #   # use system tcl/tk
 #   if((${PYTHON_VERSION_STRING} VERSION_EQUAL "3.7") OR (${PYTHON_VERSION_STRING} VERSION_GREATER "3.7"))
 #     # fetch tcl/tk sources to match the one used in Python 3.7
@@ -124,14 +189,19 @@ elseif(WIN32)
     BUILD_IN_SOURCE 1
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
-    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory . ${CMAKE_INSTALL_PREFIX}
-    LOG_DOWNLOAD 1
+    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory lib ${CMAKE_INSTALL_PREFIX}/${NG_INSTALL_DIR_LIB}
+	    COMMAND ${CMAKE_COMMAND} -E copy_directory bin ${CMAKE_INSTALL_PREFIX}/${NG_INSTALL_DIR_BIN}
+	    COMMAND ${CMAKE_COMMAND} -E copy_directory include ${CMAKE_INSTALL_PREFIX}/${NG_INSTALL_DIR_INCLUDE}
+
+    ${SUBPROJECT_ARGS}
     )
 
   set (TK_INCLUDE_PATH ${CMAKE_INSTALL_PREFIX}/include)
   set (TCL_INCLUDE_PATH ${CMAKE_INSTALL_PREFIX}/include)
   set (TCL_LIBRARY ${CMAKE_INSTALL_PREFIX}/lib/tcl86t.lib)
   set (TK_LIBRARY ${CMAKE_INSTALL_PREFIX}/lib/tk86t.lib)
+  set (TCL_STUB_LIBRARY ${CMAKE_INSTALL_PREFIX}/lib/tclstub86.lib)
+  set (TK_STUB_LIBRARY ${CMAKE_INSTALL_PREFIX}/lib/tkstub86.lib)
 
   list(APPEND NETGEN_DEPENDENCIES project_win_tcltk)
 else(WIN32)
@@ -148,6 +218,8 @@ else(WIN32)
 # )
 # list(APPEND NETGEN_DEPENDENCIES project_tkdnd)
 endif(APPLE)
+endif(SKBUILD)
+endif(LINUX)
 
 # Propagate settings to Netgen subproject
-set_vars(NETGEN_CMAKE_ARGS TCL_INCLUDE_PATH TCL_LIBRARY TK_LIBRARY TK_INCLUDE_PATH TCL_TCLSH TK_WISH)
+set_vars(NETGEN_CMAKE_ARGS TCL_INCLUDE_PATH TCL_STUB_LIBRARY TCL_LIBRARY TK_STUB_LIBRARY TK_LIBRARY TK_INCLUDE_PATH TCL_TCLSH TK_WISH)

cmake/external_projects/zlib.cmake

@@ -6,7 +6,9 @@ if(WIN32)
     BUILD_IN_SOURCE 1
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
-    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory . ${CMAKE_INSTALL_PREFIX}
+    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory lib ${CMAKE_INSTALL_PREFIX}/${NG_INSTALL_DIR_LIB}
+	    COMMAND ${CMAKE_COMMAND} -E copy_directory bin ${CMAKE_INSTALL_PREFIX}/${NG_INSTALL_DIR_BIN}
+	    COMMAND ${CMAKE_COMMAND} -E copy_directory include ${CMAKE_INSTALL_PREFIX}/${NG_INSTALL_DIR_INCLUDE}
     LOG_DOWNLOAD 1
     )
 

cmake/generate_version_file.cmake

@@ -2,19 +2,33 @@ if(NOT BDIR)
   set(BDIR ${CMAKE_CURRENT_BINARY_DIR})
 endif()
 
-find_package(Git REQUIRED)
-
-if(GIT_FOUND AND EXISTS ${CMAKE_CURRENT_LIST_DIR}/../.git)
-  execute_process(COMMAND git describe --tags --match "v[0-9]*" --long --dirty WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} OUTPUT_VARIABLE git_version_string)
+if(NETGEN_VERSION_GIT)
+    set(git_version_string ${NETGEN_VERSION_GIT})
 else()
-  # for source package files (generated for ubuntu builds on launchpad) read the version from version.txt
+    find_package(Git REQUIRED)
+    execute_process(COMMAND git describe --tags --match "v[0-9]*" --long --dirty
+        WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+        OUTPUT_VARIABLE git_version_string
+        RESULT_VARIABLE status
+        ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+endif()
+
+if(status AND NOT status EQUAL 0)
   if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/../version.txt)
-    file(READ ${CMAKE_CURRENT_LIST_DIR}/../version.txt git_version_string )
+    # for source package files (generated for ubuntu builds on launchpad) read the version from version.txt
+    if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/../version.txt)
+      file(READ ${CMAKE_CURRENT_LIST_DIR}/../version.txt git_version_string )
+    else()
+      get_filename_component(git_version_string ${CMAKE_CURRENT_LIST_DIR}/.. NAME)
+      string(REGEX REPLACE "^netgen(.*)" "\\1" git_version_string "${git_version_string}")
+    endif()
   else()
-    get_filename_component(git_version_string ${CMAKE_CURRENT_LIST_DIR}/.. NAME)
-    string(REGEX REPLACE "^netgen(.*)" "\\1" git_version_string "${git_version_string}")
+    MESSAGE(WARNING "Could not determine git-version from source code - assuming 6.2.0-0")
+    set(git_version_string "v6.2.0-0")
   endif()
 endif()
+string(STRIP ${git_version_string} git_version_string)
 
 string(REGEX REPLACE "^v([0-9]+)\\..*" "\\1" NETGEN_VERSION_MAJOR "${git_version_string}")
 string(REGEX REPLACE "^v[0-9]+\\.([0-9]+).*" "\\1" NETGEN_VERSION_MINOR "${git_version_string}")
@@ -35,8 +49,27 @@ endif()
 
 set(NETGEN_VERSION_LONG ${NETGEN_VERSION_SHORT}-${NETGEN_VERSION_TWEAK}-${NETGEN_VERSION_HASH})
 
+if(NOT NETGEN_VERSION_GIT)
+    set(NETGEN_VERSION_GIT ${NETGEN_VERSION_LONG})
+endif()
+
+if(NOT NETGEN_VERSION_PYTHON)
+    set(NETGEN_VERSION_PYTHON ${NETGEN_VERSION_TWEAK})
+endif()
+
+
 set(version_file ${BDIR}/netgen_version.hpp)
-set(new_version_file_string "#define NETGEN_VERSION \"${NETGEN_VERSION}\"\n")
+set(new_version_file_string "\
+#ifndef NETGEN_VERSION_HPP_INCLUDED
+#define NETGEN_VERSION_HPP_INCLUDED
+#define NETGEN_VERSION \"${NETGEN_VERSION}\"
+#define NETGEN_VERSION_MAJOR ${NETGEN_VERSION_MAJOR}
+#define NETGEN_VERSION_MINOR ${NETGEN_VERSION_MINOR}
+#define NETGEN_VERSION_PATCH ${NETGEN_VERSION_PATCH}
+#define NETGEN_VERSION_TWEAK ${NETGEN_VERSION_TWEAK}
+#define NETGEN_VERSION_HASH \"${NETGEN_VERSION_HASH}\"
+#endif // NETGEN_VERSION_HPP_INCLUDED
+")
 if(EXISTS ${version_file})
   file(READ ${version_file} old_version_file_string )
   if(${old_version_file_string} STREQUAL ${new_version_file_string})
@@ -47,3 +80,33 @@ else()
     file(WRITE ${BDIR}/netgen_version.hpp ${new_version_file_string})
 endif()
 
+file(GENERATE OUTPUT netgen_config.hpp CONTENT
+"\
+#ifndef NETGEN_CONFIG_HPP_INCLUDED___
+#define NETGEN_CONFIG_HPP_INCLUDED___
+
+#define NETGEN_USE_NATIVE_ARCH          $<BOOL:${USE_NATIVE_ARCH}>
+#define NETGEN_USE_GUI                  $<BOOL:${USE_GUI}>
+#define NETGEN_USE_PYTHON               $<BOOL:${USE_PYTHON}>
+#define NETGEN_USE_MPI                  $<BOOL:${USE_MPI}}>
+#define NETGEN_USE_MPI4PY               $<BOOL:${USE_MPI4PY}>
+#define NETGEN_USE_OCC                  $<BOOL:${USE_OCC}}>
+#define NETGEN_USE_JPEG                 $<BOOL:${USE_JPEG}}>
+#define NETGEN_USE_MPEG                 $<BOOL:${USE_MPEG}}>
+#define NETGEN_USE_CGNS                 $<BOOL:${USE_CGNS}}>
+#define NETGEN_USE_NUMA                 $<BOOL:${USE_NUMA}}>
+#define NETGEN_INTEL_MIC                $<BOOL:${USE_INTEL_MIC}}>
+#define NETGEN_INSTALL_PROFILES         $<BOOL:${INSTALL_PROFILES}>
+#define NETGEN_USE_CCACHE               $<BOOL:${USE_CCACHE}}>
+#define NETGEN_USE_INTERNAL_TCL         $<BOOL:${USE_INTERNAL_TCL}>
+#define NETGEN_ENABLE_UNIT_TESTS        $<BOOL:${ENABLE_UNIT_TESTS}>
+#define NETGEN_ENABLE_CPP_CORE_GUIDELINES_CHECK $<BOOL:${ENABLE_CPP_CORE_GUIDELINES_CHECK}>
+#define NETGEN_USE_SPDLOG               $<BOOL:${USE_SPDLOG}>
+#define NETGEN_DEBUG_LOG                $<BOOL:${DEBUG_LOG}>
+#define NETGEN_USE_CHECK_RANGE          $<BOOL:${CHECK_RANGE}>
+#define NETGEN_BUILD_STUB_FILES         $<BOOL:${BUILD_STUB_FILES}>
+#define NETGEN_BUILD_FOR_CONDA          $<BOOL:${BUILD_FOR_CONDA}>
+#define NETGEN_SHARED_LIBRARY_SUFFIX    \"${CMAKE_SHARED_LIBRARY_SUFFIX}\"
+
+#endif // NETGEN_CONFIG_HPP_INCLUDED___
+")

doc/element_types.tex

@@ -0,0 +1,282 @@
+\documentclass[convert=pdf2svg]{standalone}
+% \documentclass{article}
+
+\usepackage[T1]{fontenc}
+\usepackage{lmodern}
+\renewcommand{\familydefault}{\sfdefault}
+\usepackage{tikz}
+\usepackage{tikz-3dplot}
+\usetikzlibrary{external}
+\tikzset{external/force remake}
+\tikzset{external/disable dependency files}
+\tikzset{external/aux in dpth={false}}
+% uncomment this to generate a figure for each cell type (and change documentclass to article)
+% \tikzexternalize
+
+\tikzstyle{vertex} = [circle,draw=black,fill=black,scale = 0.5]
+\tdplotsetmaincoords{70}{110}
+
+% cnode(tag,x,y,z,label,label_pos)
+\def\cnode(#1,#2,#3,#4,#5,#6){
+\node (#1) at (#2,#3,#4) [vertex,label=#6:$\mathsf{#5}$] {};
+}
+
+\pagestyle{empty}
+
+\begin{document}
+
+\begin{tabular}{cc}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+SEGMENT &
+SEGMENT3
+\\
+\tikzsetnextfilename{line}
+\begin{tikzpicture}[scale = 2]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,2,below right);
+\draw (n0) -- (n1);
+\end{tikzpicture}
+&
+\tikzsetnextfilename{line3}
+\begin{tikzpicture}[scale = 2]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,2,below right);
+\cnode(n2,1,0,0,3,below right);
+\draw (n0) -- (n2) -- (n1);
+\end{tikzpicture}
+\\[1 em]
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+TRIG &
+TRIG6
+\\
+\tikzsetnextfilename{triangle}
+\begin{tikzpicture}[scale = 2]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,2,below right);
+\cnode(n2,0,2,0,3,right);
+\draw (n0) -- (n1) -- (n2) -- (n0);
+\end{tikzpicture}
+&
+\tikzsetnextfilename{triangle6}
+\begin{tikzpicture}[scale = 2]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,2,below right);
+\cnode(n2,0,2,0,3,right);
+\cnode(n3,1,0,0,6,below right);
+\cnode(n4,1,1,0,4,right);
+\cnode(n5,0,1,0,5,below right);
+\draw (n0) -- (n3) -- (n1) -- (n4) -- (n2) -- (n5) -- (n0);
+\end{tikzpicture}
+\\[1 em]
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+QUAD &
+QUAD8
+\\
+\tikzsetnextfilename{quad}
+\begin{tikzpicture}[scale = 2]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,2,below right);
+\cnode(n2,2,2,0,3,below right);
+\cnode(n3,0,2,0,4,below right);
+\draw (n0) -- (n1) -- (n2) -- (n3) -- (n0);
+\end{tikzpicture}
+&
+\tikzsetnextfilename{quad8}
+\begin{tikzpicture}[scale = 2]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,2,below right);
+\cnode(n2,2,2,0,3,below right);
+\cnode(n3,0,2,0,4,below right);
+\cnode(n4,1,0,0,5,below right);
+\cnode(n5,2,1,0,8,below right);
+\cnode(n6,1,2,0,6,below right);
+\cnode(n7,0,1,0,7,below right);
+\draw (n0) -- (n4) -- (n1) -- (n5) -- (n2) -- (n6) -- (n3) -- (n7) -- (n0);
+\end{tikzpicture}
+\\[1 em]
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+TET &
+TET10
+\\
+\tikzsetnextfilename{tetra}
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,3,below right);
+\cnode(n2,0,2,0,2,below right);
+\cnode(n3,0,0,2,4,right);
+\draw (n0) -- (n1) -- (n2) -- (n0);
+\draw (n0) -- (n3);
+\draw (n1) -- (n3);
+\draw (n2) -- (n3);
+\end{tikzpicture}
+&
+\tikzsetnextfilename{tetra10} % VTK
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,3,below right);
+\cnode(n2,0,2,0,2,below right);
+\cnode(n3,0,0,2,4,right);
+\cnode(n4,1,0,0,6,below right);
+\cnode(n5,1,1,0,8,below right);
+\cnode(n6,0,1,0,5,below right);
+\cnode(n7,0,0,1,7,below right);
+\cnode(n8,1,0,1,10,below right);
+\cnode(n9,0,1,1,9,right);
+\draw (n0) -- (n4) -- (n1) -- (n5) -- (n2) -- (n6) -- (n0);
+\draw (n0) -- (n7) -- (n3);
+\draw (n1) -- (n8) -- (n3);
+\draw (n2) -- (n9) -- (n3);
+\end{tikzpicture}
+\\[1 em]
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+PYRAMID &
+PYRAMID13
+\\
+\tikzsetnextfilename{pyramid}
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,4,below right);
+\cnode(n2,2,2,0,3,below right);
+\cnode(n3,0,2,0,2,below right);
+\cnode(n4,1,1,2,5,right);
+\draw (n0) -- (n1) -- (n2) -- (n3) -- (n0);
+\draw (n0) -- (n4);
+\draw (n1) -- (n4);
+\draw (n2) -- (n4);
+\draw (n3) -- (n4);
+\end{tikzpicture}
+&
+\tikzsetnextfilename{pyramid13} % VTK != gmsh
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,4,below right);
+\cnode(n2,2,2,0,3,below right);
+\cnode(n3,0,2,0,2,below right);
+\cnode(n4,1,1,2,5,right);
+\cnode(n5,1,0,0,8,below right);
+\cnode(n6,2,1,0,7,below right);
+\cnode(n7,1,2,0,9,below right);
+\cnode(n8,0,1,0,6,below right);
+\cnode(n9,0.5,0.5,1,10,below right);
+\cnode(n10,1.5,0.5,1,13,below right);
+\cnode(n11,1.5,1.5,1,12,below right);
+\cnode(n12,0.5,1.5,1,11,right);
+\draw (n0) -- (n5) -- (n1) -- (n6) -- (n2) -- (n7) -- (n3) -- (n8) -- (n0);
+\draw (n0) -- (n9) -- (n4);
+\draw (n1) -- (n10) -- (n4);
+\draw (n2) -- (n11) -- (n4);
+\draw (n3) -- (n12) -- (n4);
+\end{tikzpicture}
+\\[1 em]
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+PRISM &
+PRISM15
+\\
+\tikzsetnextfilename{wedge} % gmsh != VTK
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,3,below right);
+\cnode(n2,0,2,0,2,below right);
+\cnode(n3,0,0,2,4,below right);
+\cnode(n4,2,0,2,6,below right);
+\cnode(n5,0,2,2,5,below right);
+\draw (n0) -- (n1) -- (n2) -- (n0);
+\draw (n3) -- (n4) -- (n5) -- (n3);
+\draw (n0) -- (n3);
+\draw (n1) -- (n4);
+\draw (n2) -- (n5);
+\end{tikzpicture}
+&
+\tikzsetnextfilename{wedge15} % VTK != gmsh
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,3,below right);
+\cnode(n2,0,2,0,2,below right);
+\cnode(n3,0,0,2,4,below right);
+\cnode(n4,2,0,2,6,below right);
+\cnode(n5,0,2,2,5,below right);
+\cnode(n6,1,0,0,8,below right);
+\cnode(n7,1,1,0,9,below right);
+\cnode(n8,0,1,0,7,below right);
+\cnode(n9,1,0,2,14,below right);
+\cnode(n10,1,1,2,15,below right);
+\cnode(n11,0,1,2,13,below right);
+\cnode(n12,0,0,1,10,below right);
+\cnode(n13,2,0,1,12,below right);
+\cnode(n14,0,2,1,11,below right);
+\draw (n0) -- (n6) -- (n1) -- (n7) -- (n2) -- (n8) -- (n0);
+\draw (n3) -- (n9) -- (n4) -- (n10) -- (n5) -- (n11) -- (n3);
+\draw (n0) -- (n12) -- (n3);
+\draw (n1) -- (n13) -- (n4);
+\draw (n2) -- (n14) -- (n5);
+\end{tikzpicture}
+\\[1 em]
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+HEX &
+HEX20
+\\
+\tikzsetnextfilename{hexahedron}
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,4,below right);
+\cnode(n2,2,2,0,3,below right);
+\cnode(n3,0,2,0,2,below right);
+\cnode(n4,0,0,2,5,below right);
+\cnode(n5,2,0,2,8,below right);
+\cnode(n6,2,2,2,7,below right);
+\cnode(n7,0,2,2,6,below right);
+\draw (n0) -- (n1) -- (n2) -- (n3) -- (n0);
+\draw (n4) -- (n5) -- (n6) -- (n7) -- (n4);
+\draw (n0) -- (n4);
+\draw (n1) -- (n5);
+\draw (n2) -- (n6);
+\draw (n3) -- (n7);
+\end{tikzpicture}
+&
+\tikzsetnextfilename{hexahedron20} % VTK != gmsh
+\begin{tikzpicture}[scale = 2, tdplot_main_coords]
+\cnode(n0,0,0,0,1,below right);
+\cnode(n1,2,0,0,4,below right);
+\cnode(n2,2,2,0,3,below right);
+\cnode(n3,0,2,0,2,below right);
+\cnode(n4,0,0,2,5,below right);
+\cnode(n5,2,0,2,8,below right);
+\cnode(n6,2,2,2,7,below right);
+\cnode(n7,0,2,2,6,below right);
+\cnode(n8,1,0,0,11,below right);
+\cnode(n9,2,1,0,10,below right);
+\cnode(n10,1,2,0,12,below right);
+\cnode(n11,0,1,0,9,below right);
+\cnode(n12,1,0,2,15,below right);
+\cnode(n13,2,1,2,14,below right);
+\cnode(n14,1,2,2,16,below right);
+\cnode(n15,0,1,2,13,below right);
+\cnode(n16,0,0,1,17,below right);
+\cnode(n17,2,0,1,20,below right);
+\cnode(n18,2,2,1,19,below right);
+\cnode(n19,0,2,1,18,below right);
+\draw (n0) -- (n8) -- (n1) -- (n9) -- (n2) -- (n10) -- (n3) -- (n11) -- (n0);
+\draw (n4) -- (n12) -- (n5) -- (n13) -- (n6) -- (n14) -- (n7) -- (n15) -- (n4);
+\draw (n0) -- (n16) -- (n4);
+\draw (n1) -- (n17) -- (n5);
+\draw (n2) -- (n18) -- (n6);
+\draw (n3) -- (n19) -- (n7);
+\end{tikzpicture}
+
+
+
+\end{tabular}
+
+\end{document}

doc/ng4.tex

@@ -176,7 +176,7 @@ Lines starting with $\#$ are comment lines. Every CSG file must contain the
 keyword {\tt algebraic3d} before any non-comment line.
 The keyword {\tt solid} defines a named solid, here the solid {\it cube}
 is defined. A solid is defined by the Eulerian operations applied to 
-primitives. Here, the solid is just the primitve defined by {\tt orthobrick}.
+primitives. Here, the solid is just the primitive defined by {\tt orthobrick}.
 This is a brick parallel to the axis, specified by the minimal $x$, $y$, and
 $z$ coordinates, and the maximal $x$, $y$, and $z$ coordinates. The present
 definition gives the cube $[0,1]^3$. Finally, the definition {\tt tlo cube}
@@ -260,7 +260,7 @@ amount of red, green and blue (RGB) values. The flag {\tt
   -transparent} makes the solid appear transparent.
 
 
-It is possible to specify bounday condition numbers for individual
+It is possible to specify boundary condition numbers for individual
 surfaces of a solid. The flag {\tt -bc} assigns the bc to all 
 surfaces of that solid-tree. If several flags are given the one closest
 to the leaves of the tree dominates. The following file defines a 
@@ -536,7 +536,7 @@ STL is a standardized file format to describe (approximate) geometies
 by triangulated surfaces. It is useful to describe complicated parts
 which are modeled with some CAD programmes. Also, some users have written
 their own (C) programmes to define STL geometries, where was not so easy
-to use the CSG format. The syntac of STL files is as follos
+to use the CSG format. The syntax of STL files is as follows
 \begin{quote}
 (not available yet. please figure out the syntax from the examples)
 \end{quote}
@@ -593,10 +593,10 @@ Finally, the refinement factor along the line follows.
 \chapter{Mesh and Solution Formats}
 
 You can export meshes to a couple of file formats. Some are self-defined,
-some other are standard formats. The self-defined are the followings:
+some other are standard formats. The self-defined are the following:
 
 \section{Mesh Size File}
-By means of a mesh size file you can provide a local mesh size density. The file extension must be {\it .msz}. If you want to use the mesh size file, you specify it in the ``Meshing Options'', doalog box, page ``Mesh Size''.
+By means of a mesh size file you can provide a local mesh size density. The file extension must be {\it .msz}. If you want to use the mesh size file, you specify it in the ``Meshing Options'', dialog box, page ``Mesh Size''.
 
 The syntay is:
 \begin{verbatim}
@@ -643,7 +643,7 @@ The Fepp 2D format contains the following sections:
 
 \begin{enumerate}
 \item
-boundary segmetns \\
+boundary segments \\
 After the number of boundary segments there follows a list of 
 segments. Each segment is specified by the spline - patch number,
 and the two node indices. Counting starts with 1
@@ -749,7 +749,7 @@ the bottom, and the large drawing window. The menu items will be explained in
 \item Quit \newline
 Terminate Netgen
 \item Generate mesh \newline
-Performe mesh generation 
+Perform mesh generation 
 \item Stop Meshing \newline
 Stop mesh generation 
 \item Geometry/Edges/Mesh/Solution \newline

external_dependencies/pybind11

@@ -1 +1 @@
-Subproject commit deb3cb238a9f299d7c57f810165e90a1b14b3e6f
+Subproject commit 38bf7b174875c27c1ba98bdf5a9bf13d967f14d4

libsrc/CMakeLists.txt

@@ -4,10 +4,21 @@ add_subdirectory(gprim)
 add_subdirectory(linalg)
 add_subdirectory(include)
 add_subdirectory(meshing)
-add_subdirectory(visualization)
-add_subdirectory(csg)
-add_subdirectory(geom2d)
-add_subdirectory(occ)
-add_subdirectory(stlgeom)
-add_subdirectory(interface)
-
+if(USE_OCC)
+    add_subdirectory(occ)
+endif(USE_OCC)
+if(USE_STLGEOM)
+    add_subdirectory(stlgeom)
+endif(USE_STLGEOM)
+if(USE_GUI)
+    add_subdirectory(visualization)
+endif(USE_GUI)
+if(USE_INTERFACE)
+    add_subdirectory(interface)
+endif(USE_INTERFACE)
+if(USE_CSG)
+    add_subdirectory(csg)
+endif(USE_CSG)
+if(USE_GEOM2D)
+    add_subdirectory(geom2d)
+endif(USE_GEOM2D)

libsrc/core/.clang-tidy

@@ -1,4 +1,4 @@
-Checks: '*,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers'
+Checks: '*,-cppcoreguidelines-avoid-non-const-global-variables,-llvmlibc-restrict-system-libc-headers,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers'
 CheckOptions:
         - key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
           value: 1

libsrc/core/CMakeLists.txt

@@ -1,5 +1,5 @@
 
-add_library(ngcore SHARED
+add_library(ngcore ${NGCORE_LIBRARY_TYPE}
   archive.cpp
   bitarray.cpp
   exception.cpp
@@ -11,45 +11,73 @@ add_library(ngcore SHARED
   table.cpp
   taskmanager.cpp
   utils.cpp
+  version.cpp
+  ng_mpi_wrapper.cpp
   )
 
-# Pybind11 2.3 Issue https://github.com/pybind/pybind11/issues/1604
-if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
-  target_compile_options(ngcore PUBLIC -fsized-deallocation -faligned-allocation)
+string(REPLACE "|" ";" ng_compile_flags_replace_sep "${NG_COMPILE_FLAGS}")
+target_compile_options(ngcore PUBLIC ${ng_compile_flags_replace_sep})
+
+if(EMSCRIPTEN)
+  set(PYTHON_MODULE_EXTENSION ".so")
+  target_link_options(ngcore PUBLIC -sALLOW_MEMORY_GROWTH -sENVIRONMENT=web)
+  target_compile_options(ngcore PUBLIC -sNO_DISABLE_EXCEPTION_CATCHING)
+endif()
+
+if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND USE_PYTHON)
+  # Python packages on Linux are compiled with the old ABI,
+  # make sure that the same ABI is used in plugins aswell
+  try_run(
+    ret_val can_compile
+    ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/_get_glibcxx_use_cxx11_abi.cpp
+    RUN_OUTPUT_VARIABLE use_glibcxx_cxx11_abi
+    )
+  target_compile_definitions(ngcore PUBLIC -D_GLIBCXX_USE_CXX11_ABI=${use_glibcxx_cxx11_abi})
+  try_run(
+    ret_val can_compile
+    ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/_get_gxx_abi.cpp
+    RUN_OUTPUT_VARIABLE default_cxx_abi_version
+    )
+  if(${can_compile} AND (${ret_val} EQUAL 0))
+    # Different python modules using pybind11 need to use the same C++ ABI version
+    # for compatibility
+    set(cxx_abi_version 17)
+    if(cxx_abi_version LESS default_cxx_abi_version)
+      set(cxx_abi_version ${default_cxx_abi_version})
+    endif()
+    message(STATUS "GNU C++ ABI version: ${cxx_abi_version}")
+    target_compile_options(ngcore PUBLIC "-fabi-version=${cxx_abi_version}")
+  endif()
 endif()
 
 if(USE_PYTHON)
   target_sources(ngcore PRIVATE python_ngcore.cpp)
-  target_compile_definitions(ngcore PUBLIC NETGEN_PYTHON NG_PYTHON)
+  target_compile_definitions(ngcore PUBLIC NETGEN_PYTHON NG_PYTHON PYBIND11_SIMPLE_GIL_MANAGEMENT)
 endif(USE_PYTHON)
 
 if(WIN32)
-  target_compile_options(ngcore PUBLIC /bigobj /MP /W1 /wd4068)
+  target_compile_options(ngcore PUBLIC /bigobj $<BUILD_INTERFACE:/MP;/W1;/wd4068>)
   get_WIN32_WINNT(ver)
-  target_compile_definitions(ngcore PUBLIC _WIN32_WINNT=${ver} WNT WNT_WINDOW NOMINMAX MSVC_EXPRESS _CRT_SECURE_NO_WARNINGS HAVE_STRUCT_TIMESPEC)
+  target_compile_definitions(ngcore PUBLIC _WIN32_WINNT=${ver} WNT WNT_WINDOW NOMINMAX MSVC_EXPRESS _CRT_SECURE_NO_WARNINGS HAVE_STRUCT_TIMESPEC WIN32)
   target_link_options(ngcore PUBLIC /ignore:4273 /ignore:4217 /ignore:4049)
 else(WIN32)
-  target_compile_options(ngcore PRIVATE -fvisibility=hidden)
+  target_link_libraries(ngcore PUBLIC dl)
 endif(WIN32)
 
 target_compile_definitions(ngcore PRIVATE NGCORE_EXPORTS)
 target_include_directories(ngcore INTERFACE $<INSTALL_INTERFACE:${NG_INSTALL_DIR_INCLUDE}> $<INSTALL_INTERFACE:${NG_INSTALL_DIR_INCLUDE}/include>)
 
-if(CHECK_RANGE OR CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(CHECK_RANGE)
   target_compile_definitions(ngcore PUBLIC NETGEN_ENABLE_CHECK_RANGE)
-endif(CHECK_RANGE OR CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+endif(CHECK_RANGE)
 
-if(USE_SPDLOG)
-  include_directories(${SPDLOG_INCLUDE_DIR})
-  install(DIRECTORY ${SPDLOG_INCLUDE_DIR}
-    DESTINATION ${NG_INSTALL_DIR_INCLUDE}
-    )
-  add_dependencies(ngcore project_spdlog)
-  target_compile_definitions(ngcore PUBLIC NETGEN_USE_SPDLOG)
-  if(DEBUG_LOG)
-    target_compile_definitions(ngcore PUBLIC NETGEN_LOG_DEBUG)
-  endif(DEBUG_LOG)
-endif(USE_SPDLOG)
+if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+  target_compile_definitions(ngcore PUBLIC _DEBUG NETGEN_ENABLE_CHECK_RANGE)
+endif(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+
+if(TRACE_MEMORY)
+  target_compile_definitions(ngcore PUBLIC NETGEN_TRACE_MEMORY)
+endif(TRACE_MEMORY)
 
 if(USE_NUMA)
     find_library(NUMA_LIBRARY libnuma.so)
@@ -59,12 +87,15 @@ endif(USE_NUMA)
 
 install(TARGETS ngcore DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
 
-target_link_libraries(ngcore PUBLIC netgen_mpi PRIVATE netgen_python ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(ngcore PRIVATE "$<BUILD_INTERFACE:netgen_python>" ${CMAKE_THREAD_LIBS_INIT})
 
-install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp
+install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp memtracer.hpp
   exception.hpp symboltable.hpp paje_trace.hpp utils.hpp profiler.hpp mpi_wrapper.hpp
   array.hpp taskmanager.hpp concurrentqueue.h localheap.hpp python_ngcore.hpp flags.hpp
-  xbool.hpp signal.hpp bitarray.hpp table.hpp hashtable.hpp
+  xbool.hpp signal.hpp bitarray.hpp table.hpp hashtable.hpp ranges.hpp ngstream.hpp
+  simd.hpp simd_avx.hpp simd_avx512.hpp simd_generic.hpp simd_sse.hpp simd_arm64.hpp
+  register_archive.hpp autodiff.hpp autodiffdiff.hpp
+  ng_mpi.hpp ng_mpi_generated_declarations.hpp mpi4py_pycapi.h ng_mpi_native.hpp
   DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel)
 
 if(ENABLE_CPP_CORE_GUIDELINES_CHECK)
@@ -74,9 +105,71 @@ endif(ENABLE_CPP_CORE_GUIDELINES_CHECK)
 add_dependencies(ngcore ng_generate_version_file)
 
 if(USE_PYTHON)
-  pybind11_add_module(pyngcore SHARED python_ngcore_export.cpp)
-  target_link_libraries(pyngcore PUBLIC ngcore netgen_python)
-  set_target_properties(pyngcore PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/${NETGEN_PYTHON_RPATH}")
-  install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON} COMPONENT netgen)
+  pybind11_add_module(pyngcore MODULE python_ngcore_export.cpp)
+  target_link_libraries(pyngcore PUBLIC ngcore PRIVATE netgen_python)
+  set_target_properties(pyngcore PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/../${NETGEN_PYTHON_RPATH}")
+  if(EMSCRIPTEN)
+    target_compile_definitions(pyngcore PRIVATE NGCORE_EXPORTS)
+  endif(EMSCRIPTEN)
+  install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON}/pyngcore COMPONENT netgen)
 endif(USE_PYTHON)
 
+function (build_mpi_variant)
+  set(target ng_${ARGV0})
+  set(include_dir ${ARGV1})
+  message("1Building MPI variant: ${ARGV0} ${ARGV1}")
+  add_library(${target} SHARED ng_mpi.cpp)
+  target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python)
+  target_compile_definitions(${target} PUBLIC PARALLEL NG_MPI_WRAPPER)
+  target_include_directories(${target} PRIVATE ${include_dir})
+  set_target_properties(${target} PROPERTIES PREFIX "")
+  install(TARGETS ${target} RUNTIME DESTINATION ${NG_INSTALL_DIR_BIN} LIBRARY DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen)
+endfunction()
+
+if(USE_MPI)
+  target_compile_definitions(ngcore PUBLIC PARALLEL)
+
+  message(STATUS "Found MPI version\n${MPI_C_LIBRARY_VERSION_STRING}")
+
+  if(USE_MPI_WRAPPER)
+    target_compile_definitions(ngcore PUBLIC NG_MPI_WRAPPER)
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Microsoft MPI.*")
+      set(MICROSOFT_MPI_INCLUDE_DIR ${MPI_C_HEADER_DIR})
+      set(MICROSOFT_MPI_LIBRARY ${MPI_msmpi_LIBRARY})
+    endif()
+
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*")
+      set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
+    endif()
+
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "MPICH.*")
+      set(MPICH_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
+    endif()
+
+    if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Intel.*")
+      set(INTEL_MPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
+    endif()
+
+    if(OPENMPI_INCLUDE_DIR)
+      build_mpi_variant(openmpi ${OPENMPI_INCLUDE_DIR})
+    endif()
+    if(MPICH_INCLUDE_DIR)
+      build_mpi_variant(mpich ${MPICH_INCLUDE_DIR})
+    endif()
+    if(INTEL_MPI_INCLUDE_DIR)
+      build_mpi_variant(intel_mpi ${INTEL_MPI_INCLUDE_DIR})
+      if(WIN32)
+        target_link_libraries(ng_intel_mpi PUBLIC ${INTEL_MPI_LIBRARY})
+      endif()
+    endif()
+    if(MICROSOFT_MPI_INCLUDE_DIR)
+      build_mpi_variant(microsoft_mpi ${MICROSOFT_MPI_INCLUDE_DIR})
+      target_link_libraries(ng_microsoft_mpi PUBLIC ${MICROSOFT_MPI_LIBRARY})
+    endif()
+  else()
+    target_link_libraries(ngcore PUBLIC ${MPI_C_LIBRARIES})
+    target_include_directories(ngcore PUBLIC ${MPI_C_INCLUDE_PATH})
+  endif(USE_MPI_WRAPPER)
+
+endif(USE_MPI)
+

libsrc/core/_get_glibcxx_use_cxx11_abi.cpp

@@ -0,0 +1,13 @@
+#include <iostream>
+
+int main() {
+  #ifdef _GLIBCXX_USE_CXX11_ABI
+  if(_GLIBCXX_USE_CXX11_ABI)
+    std::cout << 1;
+  else
+    std::cout << 0;
+  #else // _GLIBCXX_USE_CXX11_ABI
+    std::cout << 0;
+  #endif // _GLIBCXX_USE_CXX11_ABI
+  return 0;
+}

libsrc/core/_get_gxx_abi.cpp

@@ -0,0 +1,7 @@
+#include <iostream>
+
+int main() {
+  if (__GXX_ABI_VERSION >= 2000 || __GXX_ABI_VERSION < 1000) return 1;
+  std::cout << (__GXX_ABI_VERSION % 100);
+  return 0;
+}

libsrc/core/archive.cpp

@@ -1,5 +1,7 @@
 
 #include "archive.hpp"
+#include "register_archive.hpp"
+#include "version.hpp"
 
 #ifndef WIN32
 #include <cxxabi.h>
@@ -8,40 +10,39 @@
 namespace ngcore
 {
   // clang-tidy should ignore this static object
-  static std::map<std::string, VersionInfo> library_versions;  // NOLINT
-  std::map<std::string, VersionInfo>& Archive :: GetLibraryVersions()
+  // static std::map<std::string, detail::ClassArchiveInfo> type_register;  // NOLINT
+
+  auto& GetTypeRegister()
   {
-    return library_versions;
+    static std::map<std::string, detail::ClassArchiveInfo> type_register;
+    return type_register;
   }
-  const VersionInfo& GetLibraryVersion(const std::string& library)
-  { return library_versions[library]; }
-
-  void SetLibraryVersion(const std::string& library, const VersionInfo& version)
-  { library_versions[library] = version; }
-
-  // clang-tidy should ignore this static object
-  static std::unique_ptr<std::map<std::string, detail::ClassArchiveInfo>> type_register;  // NOLINT
+  
   const detail::ClassArchiveInfo& Archive :: GetArchiveRegister(const std::string& classname)
   {
-    if(type_register == nullptr) type_register =
-                                   std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
-    return (*type_register)[classname];
+    // if(type_register == nullptr) type_register =
+    // std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
+    return GetTypeRegister()[classname];
   }
   void Archive :: SetArchiveRegister(const std::string& classname, const detail::ClassArchiveInfo& info)
   {
-    if(type_register == nullptr) type_register =
-                                   std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
-    (*type_register)[classname] = info;
-  }
-  void Archive :: RemoveArchiveRegister(const std::string& classname)
-  {
-    if(IsRegistered(classname))
-        type_register->erase(classname);
+    // if(type_register == nullptr) type_register =
+    // std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
+    GetTypeRegister()[classname] = info;
   }
   bool Archive :: IsRegistered(const std::string& classname)
   {
-    if(type_register == nullptr) type_register =
-                                   std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
-    return type_register->count(classname) != 0;
+    // if(type_register == nullptr) type_register =
+    // std::make_unique<std::map<std::string, detail::ClassArchiveInfo>>();
+    return GetTypeRegister().count(classname) != 0;
   }
+
+#ifdef NETGEN_PYTHON
+  pybind11::object CastAnyToPy(const std::any& a)
+  {
+    auto info = Archive::GetArchiveRegister(Demangle(a.type().name()));
+    return info.anyToPyCaster(a);
+  }
+#endif // NETGEN_PYTHON
+
 } // namespace ngcore

libsrc/core/archive.hpp

@@ -1,21 +1,26 @@
 #ifndef NETGEN_CORE_ARCHIVE_HPP
 #define NETGEN_CORE_ARCHIVE_HPP
 
+#include <algorithm>
+#include <any>
 #include <array>                // for array
 #include <complex>              // for complex
 #include <cstring>              // for size_t, strlen
+#include <filesystem>           // for path
 #include <fstream>              // for ifstream, ofstream
 #include <functional>           // for function
 #include <map>                  // for map
 #include <memory>               // for shared_ptr
+#include <optional>             // for optional
 #include <string>               // for string
 #include <type_traits>          // for declval, enable_if_t, false_type, is_co...
+#include <cstddef>              // for std::byte
+#include <set>                  // for set
 #include <typeinfo>             // for type_info
 #include <utility>              // for move, swap, pair
 #include <vector>               // for vector
 
 #include "exception.hpp"        // for UnreachableCodeException, Exception
-#include "logging.hpp"          // for logger
 #include "ngcore_api.hpp"       // for NGCORE_API
 #include "type_traits.hpp"      // for all_of_tmpl
 #include "utils.hpp"            // for Demangle, unlikely
@@ -30,24 +35,77 @@ namespace pybind11
 
 namespace ngcore
 {
-  // Libraries using this archive can store their version here to implement backwards compatibility
-  NGCORE_API const VersionInfo& GetLibraryVersion(const std::string& library);
-  NGCORE_API void SetLibraryVersion(const std::string& library, const VersionInfo& version);
+  template <typename T>
+  struct Shallow {
+    T val;
+    Shallow() = default;
+    Shallow(T aval) : val(aval) { ; }
+    operator T&() { return val; }
+  };
+
+  // Helper to detect shared_from_this
+  template <typename T>
+  class has_shared_from_this2
+    {
+    private:
+      // typedef T* T_ptr;
+      template <typename C> static std::true_type test(decltype(((C*)nullptr)->shared_from_this()));
+      template <typename C> static std::false_type test(...);
+      
+    public:
+      // If the test returns true_type, then T has shared_from_this
+      static constexpr bool value = decltype(test<T>(0))::value;
+  };
+  
+  
+
+  
+  template <typename T, typename = void>
+  class has_shallow_archive : public std::false_type {};
+  
+  template <typename T>
+  class has_shallow_archive<T, std::void_t<decltype(T::shallow_archive)>>
+    : public std::is_same<decltype(T::shallow_archive), std::true_type> {};
+  
+
+  
+#ifdef NETGEN_PYTHON
+  pybind11::object CastAnyToPy(const std::any& a);
+#endif // NETGEN_PYTHON
 
   class NGCORE_API Archive;
-
   namespace detail
   {
+    template <class T, class Tuple, size_t... Is>
+    T* construct_from_tuple(Tuple&& tuple, std::index_sequence<Is...> ) {
+      // return new T{std::get<Is>(std::forward<Tuple>(tuple))...};
+      return new T{std::get<Is>(std::move(tuple))...};
+    }
+
+    template <class T, class Tuple>
+    T* construct_from_tuple(Tuple&& tuple) {
+      return construct_from_tuple<T>(std::forward<Tuple>(tuple),
+                                     std::make_index_sequence<std::tuple_size<std::decay_t<Tuple>>::value>{}
+                                     );
+    }
+
     // create new pointer of type T if it is default constructible, else throw
-    template<typename T, typename ...Rest>
-    T* constructIfPossible_impl(Rest... /*unused*/)
-    { throw Exception(std::string(Demangle(typeid(T).name())) + " is not default constructible!"); }
+    template<typename T, typename... TArgs>
+    T* constructIfPossible(std::tuple<TArgs...> args)
+    {
+      if constexpr(std::is_constructible_v<T, TArgs...>)
+        return construct_from_tuple<T>(args);
+          throw Exception(std::string(Demangle(typeid(T).name())) +
+                          " is not constructible!");
+    }
 
-    template<typename T, typename= std::enable_if_t<std::is_constructible<T>::value>>
-    T* constructIfPossible_impl(int /*unused*/) { return new T; } // NOLINT
-
-    template<typename T>
-    T* constructIfPossible() { return constructIfPossible_impl<T>(int{}); }
+    template <typename T> T *constructIfPossible()
+    {
+      if constexpr(std::is_constructible_v<T>)
+        return new T();
+      throw Exception(std::string(Demangle(typeid(T).name())) +
+                      " is not default constructible!");
+    }
 
     //Type trait to check if a class implements a 'void DoArchive(Archive&)' function
     template<typename T>
@@ -79,23 +137,71 @@ namespace ngcore
       NGCORE_API static constexpr bool value = type::value;
     };
 
+    template <typename T>
+    struct has_GetCArgs
+    {
+      template <typename C> static std::true_type check( decltype( sizeof(&C::GetCArgs )) ) { return std::true_type(); }
+      template <typename> static std::false_type check(...) { return std::false_type(); }
+      typedef decltype( check<T>(sizeof(char)) ) type;
+      static constexpr type value = type();
+    };
+    template<typename T>
+    constexpr bool has_GetCArgs_v = has_GetCArgs<T>::value;
+
+    template<typename T,
+    typename std::enable_if<!has_GetCArgs_v<T>>::type* = nullptr>
+    std::tuple<> GetCArgs(T&val) { return {}; }
+
+    template<typename T,
+    typename std::enable_if<has_GetCArgs_v<T>>::type* = nullptr>
+    auto GetCArgs(T&val) {
+      return val.GetCArgs();
+    }
+
+    template<typename T>
+    using TCargs = decltype(GetCArgs<T>(*static_cast<T*>(nullptr)));
+
+
     struct ClassArchiveInfo
     {
       // create new object of this type and return a void* pointer that is points to the location
       // of the (base)class given by type_info
-      std::function<void*(const std::type_info&)> creator;
+      // std::function<void*(const std::type_info&)> creator;
+      void* (*creator)(const std::type_info&, Archive&);
       // This caster takes a void* pointer to the type stored in this info and casts it to a
       // void* pointer pointing to the (base)class type_info
-      std::function<void*(const std::type_info&, void*)> upcaster;
+      // std::function<void*(const std::type_info&, void*)> upcaster;
+      void* (*upcaster) (const std::type_info&, void*);
       // This caster takes a void* pointer to the (base)class type_info and returns void* pointing
       // to the type stored in this info
-      std::function<void*(const std::type_info&, void*)> downcaster;
+      // std::function<void*(const std::type_info&, void*)> downcaster;
+      void* (*downcaster)(const std::type_info&, void*);
+
+      // Archive constructor arguments
+      // std::function<void(Archive&, void*)> cargs_archiver;
+      void (*cargs_archiver)(Archive&, void*);
+
+#ifdef NETGEN_PYTHON
+      // std::function<pybind11::object(const std::any&)> anyToPyCaster;
+      pybind11::object (*anyToPyCaster)(const std::any&);
+#endif // NETGEN_PYTHON
     };
   } // namespace detail
 
   template<typename T>
   constexpr bool is_archivable = detail::is_Archivable_struct<T>::value;
 
+  
+  template <typename T, typename ... Trest>
+  constexpr size_t TotSize () 
+  {
+    if constexpr (sizeof...(Trest) == 0)
+                   return sizeof(T);
+    else
+      return sizeof(T) + TotSize<Trest...> ();
+  }
+  
+  
   // Base Archive class
   class NGCORE_API Archive
   {
@@ -110,8 +216,10 @@ namespace ngcore
   protected:
     bool shallow_to_python = false;
     std::map<std::string, VersionInfo> version_map = GetLibraryVersions();
-    std::shared_ptr<Logger> logger = GetLogger("Archive");
   public:
+    template<typename T>
+      static constexpr bool is_archivable = detail::is_Archivable_struct<T>::value;
+
     Archive() = delete;
     Archive(const Archive&) = delete;
     Archive(Archive&&) = delete;
@@ -156,6 +264,8 @@ namespace ngcore
     virtual void NeedsVersion(const std::string& /*unused*/, const std::string& /*unused*/) {}
 
     // Pure virtual functions that have to be implemented by In-/OutArchive
+    virtual Archive & operator & (std::byte & d) = 0;    
+    virtual Archive & operator & (float & d) = 0;
     virtual Archive & operator & (double & d) = 0;
     virtual Archive & operator & (int & i) = 0;
     virtual Archive & operator & (long & i) = 0;
@@ -207,7 +317,7 @@ namespace ngcore
       Do(&v[0], size);
       return (*this);
     }
-
+ 
     // archive implementation for enums
     template<typename T>
     auto operator & (T& val) -> std::enable_if_t<std::is_enum<T>::value, Archive&>
@@ -226,7 +336,6 @@ namespace ngcore
     // don't use it that often anyway)
     Archive& operator& (std::vector<bool>& v)
     {
-      logger->debug("In special archive for std::vector<bool>");
       size_t size;
       if(Output())
         size = v.size();
@@ -271,12 +380,53 @@ namespace ngcore
         }
       return (*this);
     }
+    template<typename T>
+    Archive& operator& (std::optional<T>& opt)
+    {
+        bool has_value = opt.has_value();
+        (*this) & has_value;
+        if(has_value)
+          {
+            if(Output())
+                (*this) << *opt;
+            else
+            {
+                T value;
+                (*this) & value;
+                opt = value;
+            }
+          }
+      return (*this);
+    }
+    template <typename T>
+    Archive& operator&(std::set<T> &s)
+    {
+      auto size = s.size();
+      (*this) & size;
+      if(Output())
+        for(const auto & val : s)
+          (*this) << val;
+      else
+      {
+          for(size_t i=0; i<size; i++)
+          {
+              T val;
+              (*this) & val;
+              s.insert(val);
+          }
+      }
+      return *this;
+    }
+
     // Archive arrays =====================================================
     // this functions can be overloaded in Archive implementations for more efficiency
     template <typename T, typename = std::enable_if_t<is_archivable<T>>>
     Archive & Do (T * data, size_t n)
     { for (size_t j = 0; j < n; j++) { (*this) & data[j]; }; return *this; }; // NOLINT
 
+    virtual Archive & Do (std::byte * d, size_t n)
+    { for (size_t j = 0; j < n; j++) { (*this) & d[j]; }; return *this; }; // NOLINT
+
     virtual Archive & Do (double * d, size_t n)
     { for (size_t j = 0; j < n; j++) { (*this) & d[j]; }; return *this; }; // NOLINT
 
@@ -305,28 +455,83 @@ namespace ngcore
       val.DoArchive(*this); return *this;
     }
 
+
+
+    
+    // pack elements to binary
+    template <typename ... Types>
+      Archive & DoPacked (Types & ... args)
+    {
+      if (true) // (isbinary)
+        {
+          constexpr size_t totsize = TotSize<Types...>(); // (args...);
+          std::byte mem[totsize];
+          if (is_output)
+            {
+              CopyToBin (&mem[0], args...);
+              Do(&mem[0], totsize);
+            }
+          else
+            {
+              Do(&mem[0], totsize);
+              CopyFromBin (&mem[0], args...);
+            }
+        }
+      // else
+      // cout << "DoPacked of non-binary called --> individual pickling" << endl;
+      return *this;
+    }
+    
+    
+    template <typename T, typename ... Trest>
+      constexpr void CopyToBin (std::byte * ptr, T & first, Trest & ...rest) const
+    {
+      memcpy (ptr, &first, sizeof(first));
+      CopyToBin(ptr+sizeof(first), rest...);
+    }
+    constexpr void CopyToBin (std::byte * ptr) const { }
+    
+    template <typename T, typename ... Trest>
+      constexpr void CopyFromBin (std::byte * ptr, T & first, Trest & ...rest) const
+    {
+      memcpy (&first, ptr, sizeof(first));
+      CopyFromBin(ptr+sizeof(first), rest...);
+    }
+    constexpr void CopyFromBin (std::byte * ptr) const { }
+
+
+      
+
+    template <typename T>
+    Archive& operator & (ngcore::Shallow<T>& shallow)
+    {
+      this->Shallow(shallow.val);
+      return *this;
+    }
+      
+
     // Archive shared_ptrs =================================================
     template <typename T>
     Archive& operator & (std::shared_ptr<T>& ptr)
     {
+      if constexpr(has_shallow_archive<T>::value)
+        if (shallow_to_python)
+          {
+            Shallow (ptr);
+            return *this;
+          }
+          
       if(Output())
         {
-          logger->debug("Store shared ptr of type {}", Demangle(typeid(T).name()));
           // save -2 for nullptr
           if(!ptr)
-            {
-              logger->debug("Storing nullptr");
-              return (*this) << -2;
-            }
+            return (*this) << -2;
 
           void* reg_ptr = ptr.get();
           bool neededDowncast = false;
           // Downcasting is only possible for our registered classes
           if(typeid(T) != typeid(*ptr))
             {
-              logger->debug("Typids are different: {} vs {}",
-                            Demangle(typeid(T).name()),
-                            Demangle(typeid(*ptr).name()));
               if(!IsRegistered(Demangle(typeid(*ptr).name())))
                   throw Exception(std::string("Archive error: Polymorphic type ")
                                   + Demangle(typeid(*ptr).name())
@@ -334,17 +539,12 @@ namespace ngcore
               reg_ptr = GetArchiveRegister(Demangle(typeid(*ptr).name())).downcaster(typeid(T), ptr.get());
               // if there was a true downcast we have to store more information
               if(reg_ptr != static_cast<void*>(ptr.get()))
-                {
-                  logger->debug("Multiple/Virtual inheritance involved, need to cast pointer");
-                  neededDowncast = true;
-                }
+                neededDowncast = true;
             }
           auto pos = shared_ptr2nr.find(reg_ptr);
           // if not found store -1 and the pointer
           if(pos == shared_ptr2nr.end())
             {
-              logger->debug("Didn't find the shared_ptr, create new registry entry at {}",
-                            shared_ptr_count);
               auto p = ptr.get();
               (*this) << -1;
               (*this) & neededDowncast & p;
@@ -355,27 +555,23 @@ namespace ngcore
               return *this;
             }
           // if found store the position and if it has to be downcasted and how
-          logger->debug("Found shared_ptr at position {}", pos->second);
           (*this) << pos->second << neededDowncast;
           if(neededDowncast)
             (*this) << Demangle(typeid(*ptr).name());
         }
       else // Input
         {
-          logger->debug("Reading shared_ptr of type {}", Demangle(typeid(T).name()));
           int nr;
           (*this) & nr;
           // -2 restores a nullptr
           if(nr == -2)
             {
-              logger->debug("Reading a nullptr");
               ptr = nullptr;
               return *this;
             }
           // -1 restores a new shared ptr by restoring the inner pointer and creating a shared_ptr to it
           if (nr == -1)
             {
-              logger->debug("Createing new shared_ptr");
               T* p = nullptr;
               bool neededDowncast;
               (*this) & neededDowncast & p;
@@ -383,7 +579,6 @@ namespace ngcore
               // if we did downcast we need to store a shared_ptr<void> to the true object
               if(neededDowncast)
                 {
-                  logger->debug("Shared pointer needed downcasting");
                   std::string name;
                   (*this) & name;
                   auto info = GetArchiveRegister(name);
@@ -394,20 +589,15 @@ namespace ngcore
                                                                                 ptr.get())));
                 }
               else
-                {
-                  logger->debug("Shared pointer didn't need downcasting");
                   nr2shared_ptr.push_back(ptr);
-                }
             }
           else
             {
-              logger->debug("Reading already existing pointer at entry {}", nr);
               auto other = nr2shared_ptr[nr];
               bool neededDowncast;
               (*this) & neededDowncast;
               if(neededDowncast)
                 {
-                  logger->debug("Shared pointer needed pointer downcast");
                   // if there was a downcast we can expect the class to be registered (since archiving
                   // wouldn't have worked else)
                   std::string name;
@@ -421,7 +611,6 @@ namespace ngcore
                 }
               else
                 {
-                  logger->debug("Shared pointer didn't need pointer casts");
                   ptr = std::static_pointer_cast<T>(other);
                 }
             }
@@ -435,45 +624,39 @@ namespace ngcore
     {
       if (Output())
         {
-          logger->debug("Store pointer of type {}",Demangle(typeid(T).name()));
           // if the pointer is null store -2
           if (!p)
-            {
-              logger->debug("Storing nullptr");
               return (*this) << -2;
-            }
           auto reg_ptr = static_cast<void*>(p);
           if(typeid(T) != typeid(*p))
             {
-              logger->debug("Typeids are different: {} vs {}",
-                            Demangle(typeid(T).name()),
-                            Demangle(typeid(*p).name()));
               if(!IsRegistered(Demangle(typeid(*p).name())))
                 throw Exception(std::string("Archive error: Polymorphic type ")
                                 + Demangle(typeid(*p).name())
                                 + " not registered for archive");
               reg_ptr = GetArchiveRegister(Demangle(typeid(*p).name())).downcaster(typeid(T), static_cast<void*>(p));
-              if(reg_ptr != static_cast<void*>(p))
-                {
-                  logger->debug("Multiple/Virtual inheritance involved, need to cast pointer");
-                }
             }
           auto pos = ptr2nr.find(reg_ptr);
           // if the pointer is not found in the map create a new entry
           if (pos == ptr2nr.end())
             {
-              logger->debug("Didn't find pointer, create new registry entry at {}",
-                            ptr_count);
               ptr2nr[reg_ptr] = ptr_count++;
               if(typeid(*p) == typeid(T))
                 if (std::is_constructible<T>::value)
-                               {
-                                 logger->debug("Store standard class pointer (no virt. inh,...)");
-                                 return (*this) << -1 & (*p);
-                               }
+                  return (*this) << -1 & (*p);
                 else
-                  throw Exception(std::string("Archive error: Class ") +
-                                  Demangle(typeid(*p).name()) + " does not provide a default constructor!");
+                  {
+                    if (IsRegistered(Demangle(typeid(*p).name())))
+                    {
+                      (*this) << -3 << Demangle(typeid(*p).name());
+                      GetArchiveRegister(Demangle(typeid(*p).name())).
+                        cargs_archiver(*this, p);
+                      return (*this) & (*p);
+                    }
+                    else
+                      throw Exception(std::string("Archive error: Class ") +
+                                      Demangle(typeid(*p).name()) + " does not provide a default constructor!");
+                  }
               else
                 {
                   // if a pointer to a base class is archived, the class hierarchy must be registered
@@ -484,49 +667,41 @@ namespace ngcore
                     throw Exception(std::string("Archive error: Polymorphic type ")
                                     + Demangle(typeid(*p).name())
                                     + " not registered for archive");
-                  logger->debug("Store a possibly more complicated pointer");
-                  return (*this) << -3 << Demangle(typeid(*p).name()) & (*p);
+                  (*this) << -3 << Demangle(typeid(*p).name());
+                  GetArchiveRegister(Demangle(typeid(*p).name())).
+                    cargs_archiver(*this, p);
+                  return (*this) & (*p);
                 }
             }
           else
             {
               (*this) & pos->second;
               bool downcasted = !(reg_ptr == static_cast<void*>(p) );
-              logger->debug("Store a the existing position in registry at {}", pos->second);
-              logger->debug("Pointer {} downcasting", downcasted ? "needs" : "doesn't need");
               // store if the class has been downcasted and the name
               (*this) << downcasted << Demangle(typeid(*p).name());
             }
         }
       else
         {
-          logger->debug("Reading pointer of type {}", Demangle(typeid(T).name()));
           int nr;
           (*this) & nr;
           if (nr == -2) // restore a nullptr
-            {
-              logger->debug("Loading a nullptr");
               p = nullptr;
-            }
           else if (nr == -1) // create a new pointer of standard type (no virtual or multiple inheritance,...)
             {
-              logger->debug("Load a new pointer to a simple class");
               p = detail::constructIfPossible<T>();
               nr2ptr.push_back(p);
               (*this) & *p;
             }
           else if(nr == -3) // restore one of our registered classes that can have multiple inheritance,...
             {
-              logger->debug("Load a new pointer to a potentially more complicated class "
-                            "(allows for multiple/virtual inheritance,...)");
               // As stated above, we want this special behaviour only for our classes that implement DoArchive
               std::string name;
               (*this) & name;
-              logger->debug("Name = {}", name);
               auto info = GetArchiveRegister(name);
               // the creator creates a new object of type name, and returns a void* pointing
               // to T (which may have an offset)
-              p = static_cast<T*>(info.creator(typeid(T)));
+              p = static_cast<T*>(info.creator(typeid(T), *this));
               // we store the downcasted pointer (to be able to find it again from
               // another class in a multiple inheritance tree)
               nr2ptr.push_back(info.downcaster(typeid(T),p));
@@ -534,11 +709,9 @@ namespace ngcore
             }
           else
             {
-              logger->debug("Restoring pointer to already existing object at registry position {}", nr);
               bool downcasted;
               std::string name;
               (*this) & downcasted & name;
-              logger->debug("{} object of type {}", downcasted ? "Downcasted" : "Not downcasted", name);
               if(downcasted)
                 {
                   // if the class has been downcasted we can assume it is in the register
@@ -552,6 +725,16 @@ namespace ngcore
       return *this;
     }
 
+    Archive& operator&(std::tuple<>&) { return *this; }
+
+    template <typename... T>
+    Archive& operator&(std::tuple<T...> &t)
+    {
+      // call operator& for each element of the tuple
+      std::apply([this](auto&... arg) { std::make_tuple(((*this) & arg).IsParallel()...);}, t);
+      return *this;
+    }
+
     // const ptr
     template<typename T>
     Archive& operator &(const T*& t)
@@ -570,19 +753,23 @@ namespace ngcore
 
     virtual void FlushBuffer() {}
 
-  protected:
-    static std::map<std::string, VersionInfo>& GetLibraryVersions();
-
+    bool parallel = false;
+    bool IsParallel() const { return parallel; }
+    void SetParallel (bool _parallel) { parallel = _parallel; }
+    
   private:
-    template<typename T, typename ... Bases>
+  template<typename T, typename Bases>
     friend class RegisterClassForArchive;
 
+#ifdef NETGEN_PYTHON
+    friend pybind11::object CastAnyToPy(const std::any&);
+#endif // NETGEN_PYTHON
+
     // Returns ClassArchiveInfo of Demangled typeid
     static const detail::ClassArchiveInfo& GetArchiveRegister(const std::string& classname);
     // Set ClassArchiveInfo for Demangled typeid, this is done by creating an instance of
     // RegisterClassForArchive<type, bases...>
     static void SetArchiveRegister(const std::string& classname, const detail::ClassArchiveInfo& info);
-    static void RemoveArchiveRegister(const std::string& classname);
     static bool IsRegistered(const std::string& classname);
 
     // Helper class for up-/downcasting
@@ -590,7 +777,7 @@ namespace ngcore
     struct Caster{};
 
     template<typename T>
-    struct Caster<T>
+    struct Caster<T, std::tuple<>>
     {
       static void* tryUpcast (const std::type_info& /*unused*/, T* /*unused*/)
       {
@@ -602,8 +789,37 @@ namespace ngcore
       }
     };
 
+    template<typename T, typename B1>
+    struct Caster<T,B1>
+    {
+      static void* tryUpcast(const std::type_info& ti, T* p)
+      {
+        try {
+          return GetArchiveRegister(Demangle(typeid(B1).name()))
+            .upcaster(ti, static_cast<void *>(dynamic_cast<B1 *>(p)));
+        } catch (const Exception &) {
+        throw Exception("Upcast not successful, some classes are not "
+                                "registered properly for archiving!");
+        }
+      }
+
+      static void* tryDowncast(const std::type_info& ti, void* p)
+      {
+        if(typeid(B1) == ti)
+          return dynamic_cast<T*>(static_cast<B1*>(p));
+        try
+          {
+            return dynamic_cast<T*>(static_cast<B1*>(GetArchiveRegister(Demangle(typeid(B1).name())).
+                                                     downcaster(ti, p)));
+        } catch (const Exception &) {
+            throw Exception("Downcast not successful, some classes are not "
+                            "registered properly for archiving!");
+        }
+      }
+    };
+
     template<typename T, typename B1, typename ... Brest>
-    struct Caster<T,B1,Brest...>
+    struct Caster<T,std::tuple<B1, Brest...>>
     {
       static void* tryUpcast(const std::type_info& ti, T* p)
       {
@@ -611,7 +827,7 @@ namespace ngcore
           { return GetArchiveRegister(Demangle(typeid(B1).name())).
               upcaster(ti, static_cast<void*>(dynamic_cast<B1*>(p))); }
         catch(const Exception&)
-          { return Caster<T, Brest...>::tryUpcast(ti, p); }
+          { return Caster<T, std::tuple<Brest...>>::tryUpcast(ti, p); }
       }
 
       static void* tryDowncast(const std::type_info& ti, void* p)
@@ -625,38 +841,12 @@ namespace ngcore
           }
         catch(const Exception&)
           {
-            return Caster<T, Brest...>::tryDowncast(ti, p);
+            return Caster<T, std::tuple<Brest...>>::tryDowncast(ti, p);
           }
       }
     };
   };
 
-  template<typename T, typename ... Bases>
-  class RegisterClassForArchive
-  {
-  public:
-    RegisterClassForArchive()
-    {
-      static_assert(detail::all_of_tmpl<std::is_base_of<Bases,T>::value...>,
-                    "Variadic template arguments must be base classes of T");
-      detail::ClassArchiveInfo info {};
-      info.creator = [](const std::type_info& ti) -> void*
-                     { return typeid(T) == ti ? detail::constructIfPossible<T>()
-                         : Archive::Caster<T, Bases...>::tryUpcast(ti, detail::constructIfPossible<T>()); };
-      info.upcaster = [/*this*/](const std::type_info& ti, void* p) -> void*
-                      { return typeid(T) == ti ? p : Archive::Caster<T, Bases...>::tryUpcast(ti, static_cast<T*>(p)); };
-      info.downcaster = [/*this*/](const std::type_info& ti, void* p) -> void*
-                        { return typeid(T) == ti ? p : Archive::Caster<T, Bases...>::tryDowncast(ti, p); };
-      Archive::SetArchiveRegister(std::string(Demangle(typeid(T).name())),info);
-    }
-    ~RegisterClassForArchive()
-    {
-      Archive::RemoveArchiveRegister(std::string(Demangle(typeid(T).name())));
-    }
-
-
-  };
-
   // BinaryOutArchive ======================================================================
   class NGCORE_API BinaryOutArchive : public Archive
   {
@@ -672,7 +862,7 @@ namespace ngcore
     BinaryOutArchive(std::shared_ptr<std::ostream>&& astream)
       : Archive(true), stream(std::move(astream))
     { }
-    BinaryOutArchive(const std::string& filename)
+    BinaryOutArchive(const std::filesystem::path& filename)
       : BinaryOutArchive(std::make_shared<std::ofstream>(filename)) {}
     ~BinaryOutArchive () override { FlushBuffer(); }
 
@@ -680,6 +870,10 @@ namespace ngcore
     BinaryOutArchive& operator=(BinaryOutArchive&&) = delete;
 
     using Archive::operator&;
+    Archive & operator & (std::byte & d) override
+    { return Write(d); }
+    Archive & operator & (float & f) override
+    { return Write(f); }
     Archive & operator & (double & d) override
     { return Write(d); }
     Archive & operator & (int & i) override
@@ -687,13 +881,26 @@ namespace ngcore
     Archive & operator & (short & i) override
     { return Write(i); }
     Archive & operator & (long & i) override
-    { return Write(i); }
+    {
+      // for platform independence
+      if constexpr (sizeof(long) == 8)
+        return Write(i);
+      else
+        return Write(static_cast<int64_t>(i));
+    }
     Archive & operator & (size_t & i) override
-    { return Write(i); }
+    {
+      // for platform independence
+      if constexpr (sizeof(size_t) == 8)
+        return Write(i);
+      else
+        return Write(static_cast<uint64_t>(i));
+    }
     Archive & operator & (unsigned char & i) override
     { return Write(i); }
     Archive & operator & (bool & b) override
     { return Write(b); }
+
     Archive & operator & (std::string & str) override
     {
       int len = str.length();
@@ -720,19 +927,23 @@ namespace ngcore
           ptr = 0;
         }
     }
+    Archive & Do (std::byte * d, size_t n) override
+    {
+      FlushBuffer();
+      stream->write(reinterpret_cast<char*>(d), n*sizeof(std::byte)); return *this;
+    } 
 
   private:
     template <typename T>
     Archive & Write (T x)
     {
+      static_assert(sizeof(T) < BUFFERSIZE, "Cannot write large types with this function!");
       if (unlikely(ptr > BUFFERSIZE-sizeof(T)))
         {
           stream->write(&buffer[0], ptr);
-          *reinterpret_cast<T*>(&buffer[0]) = x; // NOLINT
-          ptr = sizeof(T);
-          return *this;
+          ptr = 0;
         }
-      *reinterpret_cast<T*>(&buffer[ptr]) = x; // NOLINT
+      memcpy(&buffer[ptr], &x, sizeof(T));
       ptr += sizeof(T);
       return *this;
     }
@@ -747,10 +958,14 @@ namespace ngcore
     BinaryInArchive (std::shared_ptr<std::istream>&& astream)
       : Archive(false), stream(std::move(astream))
     { }
-    BinaryInArchive (const std::string& filename)
+    BinaryInArchive (const std::filesystem::path& filename)
       : BinaryInArchive(std::make_shared<std::ifstream>(filename)) { ; }
 
     using Archive::operator&;
+    Archive & operator & (std::byte & d) override
+    { Read(d); return *this; }
+    Archive & operator & (float & f) override
+    { Read(f); return *this; }
     Archive & operator & (double & d) override
     { Read(d); return *this; }
     Archive & operator & (int & i) override
@@ -758,9 +973,31 @@ namespace ngcore
     Archive & operator & (short & i) override
     { Read(i); return *this; }
     Archive & operator & (long & i) override
-    { Read(i); return *this; }
+    {
+      // for platform independence
+      if constexpr (sizeof(long) == 8)
+        Read(i);
+      else
+      {
+        int64_t tmp = 0;
+        Read(tmp);
+        i = tmp;
+      }
+      return *this;
+    }
     Archive & operator & (size_t & i) override
-    { Read(i); return *this; }
+    {
+      // for platform independence
+      if constexpr (sizeof(long) == 8)
+        Read(i);
+      else
+      {
+        uint64_t tmp = 0;
+        Read(tmp);
+        i = tmp;
+      }
+      return *this;
+    }
     Archive & operator & (unsigned char & i) override
     { Read(i); return *this; }
     Archive & operator & (bool & b) override
@@ -789,12 +1026,22 @@ namespace ngcore
       return *this;
     }
 
+    Archive & Do (std::byte * d, size_t n) override
+    { stream->read(reinterpret_cast<char*>(d), n*sizeof(std::byte)); return *this; } // NOLINT
     Archive & Do (double * d, size_t n) override
     { stream->read(reinterpret_cast<char*>(d), n*sizeof(double)); return *this; } // NOLINT
     Archive & Do (int * i, size_t n) override
     { stream->read(reinterpret_cast<char*>(i), n*sizeof(int)); return *this; } // NOLINT
     Archive & Do (size_t * i, size_t n) override
-    { stream->read(reinterpret_cast<char*>(i), n*sizeof(size_t)); return *this; } // NOLINT
+    {
+      // for platform independence
+      if constexpr (sizeof(long) == 8)
+        stream->read(reinterpret_cast<char*>(i), n*sizeof(size_t)); // NOLINT
+      else
+        for(size_t j = 0; j < n; j++)
+          (*this) & i[j];
+      return *this;
+    }
 
   private:
     template<typename T>
@@ -811,10 +1058,14 @@ namespace ngcore
     TextOutArchive (std::shared_ptr<std::ostream>&& astream)
       : Archive(true), stream(std::move(astream))
     { }
-    TextOutArchive (const std::string& filename) :
+    TextOutArchive (const std::filesystem::path& filename) :
       TextOutArchive(std::make_shared<std::ofstream>(filename)) { }
 
     using Archive::operator&;
+    Archive & operator & (std::byte & d) override
+    { *stream << int(d) << ' '; return *this; }
+    Archive & operator & (float & f) override
+    { *stream << f << '\n'; return *this; }
     Archive & operator & (double & d) override
     { *stream << d << '\n'; return *this; }
     Archive & operator & (int & i) override
@@ -862,10 +1113,14 @@ namespace ngcore
     TextInArchive (std::shared_ptr<std::istream>&& astream) :
       Archive(false), stream(std::move(astream))
     { }
-    TextInArchive (const std::string& filename)
+    TextInArchive (const std::filesystem::path& filename)
       : TextInArchive(std::make_shared<std::ifstream>(filename)) {}
 
     using Archive::operator&;
+    Archive & operator & (std::byte & d) override
+    { int tmp; *stream >> tmp; d = std::byte(tmp); return *this; }
+    Archive & operator & (float & f) override
+    { *stream >> f; return *this; }
     Archive & operator & (double & d) override
     { *stream >> d; return *this; }
     Archive & operator & (int & i) override
@@ -882,13 +1137,32 @@ namespace ngcore
     { char c; *stream >> c; b = (c=='t'); return *this; }
     Archive & operator & (std::string & str) override
     {
+      // Ignore \r (carriage return) characters when reading strings
+      // this is necessary for instance when a file was written on Windows and is read on Unix
+
       int len;
       *stream >> len;
       char ch;
-      stream->get(ch); // '\n'
+      stream->get(ch); // read newline character
+      if(ch == '\r') // windows line endings -> read \n as well
+        stream->get(ch);
       str.resize(len);
       if(len)
         stream->get(&str[0], len+1, '\0');
+
+      // remove all \r characters from the string, check if size changed
+      // if so, read the remaining characters
+      str.erase(std::remove(str.begin(), str.end(), '\r'), str.cend());
+      size_t chars_to_read = len-str.size();
+      while (chars_to_read>0)
+      {
+        auto old_size = str.size();
+        str.resize(len);
+
+        stream->get(&str[old_size], chars_to_read+1, '\0');
+        str.erase(std::remove(str.begin()+old_size, str.end(), '\r'), str.cend());
+        chars_to_read = len - str.size();
+      }
       return *this;
     }
     Archive & operator & (char *& str) override
@@ -905,6 +1179,8 @@ namespace ngcore
       if(len)
         {
           stream->get(ch); // \n
+          if(ch == '\r') // windows line endings, read \n as well
+            stream->get(ch);
           stream->get(&str[0], len+1, '\0'); // NOLINT
         }
       str[len] = '\0'; // NOLINT
@@ -912,6 +1188,56 @@ namespace ngcore
     }
   };
 
+  // HashArchive =================================================================
+  // This class enables to easily create hashes for archivable objects by xoring
+  // threw its data
+
+  class NGCORE_API HashArchive : public Archive
+  {
+    size_t hash_value = 0;
+    char* h;
+    int offset = 0;
+  public:
+    HashArchive() : Archive(true)
+      { h = (char*)&hash_value; }
+
+    using Archive::operator&;
+    Archive & operator & (std::byte & d) override { return ApplyHash(d); }    
+    Archive & operator & (float & f) override { return ApplyHash(f); }
+    Archive & operator & (double & d) override { return ApplyHash(d); }
+    Archive & operator & (int & i) override { return ApplyHash(i); }
+    Archive & operator & (short & i) override { return ApplyHash(i); }
+    Archive & operator & (long & i) override { return ApplyHash(i); }
+    Archive & operator & (size_t & i) override { return ApplyHash(i); }
+    Archive & operator & (unsigned char & i) override { return ApplyHash(i); }
+    Archive & operator & (bool & b) override { return ApplyHash(b); }
+    Archive & operator & (std::string & str) override
+    { for(auto c : str) ApplyHash(c);  return *this; }
+    Archive & operator & (char *& str) override
+    { char* s = str; while(*s != '\0') ApplyHash(*(s++)); return *this; }
+
+    // HashArchive can be used in const context
+    template<typename T>
+      Archive & operator& (const T& val) const
+    { return (*this) & const_cast<T&>(val); }
+
+    size_t GetHash() const { return hash_value; }
+
+  private:
+    template<typename T>
+      Archive& ApplyHash(T val)
+    {
+      size_t n = sizeof(T);
+      char* pval = (char*)&val;
+      for(size_t i = 0; i < n; i++)
+        {
+          h[offset++] ^= pval[i];
+          offset %= 8;
+        }
+      return *this;
+    }
+  };
+
 } // namespace ngcore
 
 #endif // NETGEN_CORE_ARCHIVE_HPP

libsrc/core/array.hpp

@@ -7,10 +7,15 @@
 /* Date:   01. Jun. 95                                                    */
 /**************************************************************************/
 
+#include <cstring>
+#include <type_traits>
 
-#include "archive.hpp"
 #include "exception.hpp"
+#include "logging.hpp"          // for logger
+#include "ngcore_api.hpp"       // for NGCORE_API
+#include "type_traits.hpp"      // for all_of_tmpl
 #include "localheap.hpp"
+#include "memtracer.hpp"
 #include "utils.hpp"
 
 namespace ngcore
@@ -39,7 +44,7 @@ namespace ngcore
   };
 
   template <typename ... ARGS>
-  ostream & operator<< (ostream & ost, Tuple<ARGS...> tup)
+  ostream & operator<< (ostream & ost, Tuple<ARGS...> /* tup */)
   {
     return ost;
   }
@@ -211,6 +216,24 @@ namespace ngcore
   template <typename  T>
   constexpr T IndexBASE () { return T(0); }
 
+  template <typename  T>
+  constexpr T IndexBASE (T ind) { return IndexBASE<T>(); }
+
+  
+
+  class IndexFromEnd
+  {
+    ptrdiff_t i;
+  public:
+    constexpr IndexFromEnd (ptrdiff_t ai) : i(ai) { }
+    IndexFromEnd operator+ (ptrdiff_t inc) const { return i+inc; }
+    IndexFromEnd operator- (ptrdiff_t dec) const { return i-dec; }
+    // operator ptrdiff_t () const { return i; }
+    ptrdiff_t Value() const { return i; }
+  };
+
+  constexpr IndexFromEnd END(0);
+  
   
   template <class T, class IndexType = size_t> class FlatArray;
 
@@ -259,7 +282,8 @@ namespace ngcore
     T first, next;
   public: 
     NETGEN_INLINE T_Range () { ; }
-    NETGEN_INLINE T_Range (T n) : first(0), next(n) {;}
+    // NETGEN_INLINE T_Range (T n) : first(0), next(n) {;}
+    NETGEN_INLINE explicit T_Range (size_t n) : first(IndexBASE<T>()), next(IndexBASE<T>()+n) {;}    
     NETGEN_INLINE T_Range (T f, T n) : first(f), next(n) {;}
     template <typename T2>
       NETGEN_INLINE T_Range(T_Range<T2> r2) : first(r2.First()), next(r2.Next()) { ; }
@@ -268,16 +292,16 @@ namespace ngcore
     NETGEN_INLINE T & First() { return first; }
     NETGEN_INLINE T & Next() { return next; }
     NETGEN_INLINE auto Size() const { return next-first; }
-    NETGEN_INLINE T operator[] (T i) const { return first+i; }
+    NETGEN_INLINE T operator[] (size_t i) const { return first+i; }
     NETGEN_INLINE bool Contains (T i) const { return ((i >= first) && (i < next)); }
     NETGEN_INLINE T_Range Modify(int inc_beg, int inc_end) const
     { return T_Range(first+inc_beg, next+inc_end); }
     NETGEN_INLINE ArrayRangeIterator<T> begin() const { return first; }
     NETGEN_INLINE ArrayRangeIterator<T> end() const { return next; }
 
-    T_Range Split (size_t nr, int tot) const
+    NETGEN_INLINE T_Range Split (size_t nr, int tot) const
     {
-      T diff = next-first;
+      auto diff = next-first;
       return T_Range (first + nr * diff / tot,
                       first + (nr+1) * diff / tot);
     }
@@ -421,9 +445,9 @@ namespace ngcore
   protected:
     static constexpr IndexType BASE = IndexBASE<IndexType>();
     /// the size
-    size_t size;
+    size_t size = 0;
     /// the data
-    T * __restrict data;
+    T * __restrict data = nullptr;
   public:
     typedef T value_type;
     typedef IndexType index_type;
@@ -485,7 +509,8 @@ namespace ngcore
       return *this;
     }
 
-    NETGEN_INLINE const FlatArray & operator= (const std::function<T(int)> & func) const
+    template <typename T2, std::enable_if_t<std::is_function<T2>::value>>
+    NETGEN_INLINE const FlatArray & operator= (const T2 & func) const
     {
       for (size_t i = 0; i < size; i++)
         data[i] = func(i+BASE);
@@ -534,6 +559,13 @@ namespace ngcore
     // { return CArray<T> (data+pos); }
     NETGEN_INLINE T * operator+ (size_t pos) const { return data+pos; }
 
+    /// access first element. check by macro NETGEN_CHECK_RANGE
+    T & First () const
+    {
+      NETGEN_CHECK_RANGE(0,0,size);
+      return data[0];
+    }
+
     /// access last element. check by macro NETGEN_CHECK_RANGE
     T & Last () const
     {
@@ -559,6 +591,12 @@ namespace ngcore
       return FlatArray<T> (end-start, data+start);
     }
 
+    /// takes range starting from position start of end-start elements
+    NETGEN_INLINE FlatArray<T> Range (size_t start, IndexFromEnd indend) const
+    {
+      return this->Range(start, size_t(Size()+indend.Value()));
+    }
+    
     /// takes range starting from position start of end-start elements
     NETGEN_INLINE FlatArray<T> Range (T_Range<size_t> range) const
     {
@@ -592,13 +630,31 @@ namespace ngcore
       return Pos(elem) != ILLEGAL_POSITION;
     }
     
-    auto begin() const { return ArrayIterator<T,IndexType> (*this, BASE); }
-    auto end() const { return ArrayIterator<T,IndexType> (*this, size+BASE); }
+    //auto begin() const { return ArrayIterator<T,IndexType> (*this, BASE); }
+    // auto end() const { return ArrayIterator<T,IndexType> (*this, size+BASE); }
+    NETGEN_INLINE auto begin() const { return data; }
+    NETGEN_INLINE auto end() const { return data+Size(); }
   };
 
   template <typename T>
   FlatArray<T> View (FlatArray<T> fa) { return fa; }
 
+  template <typename T, typename TI>
+  auto Max (FlatArray<T,TI> array, typename std::remove_const<T>::type max = std::numeric_limits<T>::min()) -> T
+  {
+    for (auto & v : array)
+      if (v > max) max = v;
+    return max;
+  }
+
+  template <typename T, typename TI>
+  auto Min (FlatArray<T,TI> array, typename std::remove_const<T>::type min = std::numeric_limits<T>::max()) -> T
+  {
+    for (auto & v : array)
+      if (v < min) min = v;
+    return min;
+  }
+  
   /// print array
   template <class T, class TIND>
   inline ostream & operator<< (ostream & s, const FlatArray<T, TIND> & a)
@@ -619,7 +675,13 @@ namespace ngcore
     return true;
   }
 		 
-
+  template <class T1, class T2>
+  inline bool operator!= (const FlatArray<T1> & a1,
+                          const FlatArray<T2> & a2)
+  {
+    return !(a1==a2);
+  }
+  
 
   /** 
       Dynamic array container.
@@ -637,6 +699,8 @@ namespace ngcore
     size_t allocsize;
     /// that's the data we have to delete, nullptr for not owning the memory
     T * mem_to_delete;
+    MemoryTracer mt;
+
 
     using FlatArray<T,IndexType>::size;
     using FlatArray<T,IndexType>::data;
@@ -657,6 +721,7 @@ namespace ngcore
     {
       allocsize = asize; 
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*asize);
     }
 
 
@@ -666,7 +731,10 @@ namespace ngcore
     {
       allocsize = asize;
       if(ownMemory)
+      {
         mem_to_delete = adata;
+        mt.Alloc(sizeof(T)*asize);
+      }
       else
         mem_to_delete = nullptr;
     }
@@ -682,6 +750,7 @@ namespace ngcore
 
     NETGEN_INLINE Array (Array && a2) 
     {
+      mt = std::move(a2.mt);
       size = a2.size; 
       data = a2.data;
       allocsize = a2.allocsize;
@@ -696,10 +765,20 @@ namespace ngcore
     NETGEN_INLINE explicit Array (const Array & a2)
       : FlatArray<T,IndexType> (a2.Size(), a2.Size() ? new T[a2.Size()] : nullptr)
     {
-      allocsize = size;
-      mem_to_delete = data;
-      for (size_t i = 0; i < size; i++)
-        data[i] = a2.data[i];
+      if constexpr (std::is_copy_assignable<T>::value)
+        {
+          allocsize = size;
+          mem_to_delete = data;
+          mt.Alloc(sizeof(T)*size);
+          for (size_t i = 0; i < size; i++)
+            data[i] = a2.data[i];
+        }
+      
+// #ifdef __cpp_exceptions
+#ifndef __CUDA_ARCH__
+      else
+        throw Exception(std::string("cannot copy-construct Array of type ") + typeid(T).name());
+#endif      
     }
 
     
@@ -710,6 +789,7 @@ namespace ngcore
     {
       allocsize = size;
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*size);
       /*
       for (size_t i = 0; i < size; i++)
         data[i] = a2[i];
@@ -726,6 +806,7 @@ namespace ngcore
     {
       allocsize = size;
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*size);
       size_t cnt = 0;
       for (auto val : list)
         data[cnt++] = val;
@@ -738,6 +819,7 @@ namespace ngcore
     {
       allocsize = size;
       mem_to_delete = data;
+      mt.Alloc(sizeof(T)*size);
       for(size_t i = 0; i <  a2.Size(); i++)
         data[i] = a2[i];
       for (size_t i = a2.Size(), j=0; i < size; i++,j++)
@@ -747,12 +829,15 @@ namespace ngcore
     /// if responsible, deletes memory
     NETGEN_INLINE ~Array()
     {
+      if(mem_to_delete)
+        mt.Free(sizeof(T)*allocsize);
       delete [] mem_to_delete;
     }
 
     // Only provide this function if T is archivable
-    template<typename T2=T>
-    auto DoArchive(Archive& archive) -> typename std::enable_if_t<is_archivable<T2>, void>
+    template<typename ARCHIVE>
+    auto DoArchive(ARCHIVE& archive)
+      -> typename std::enable_if_t<ARCHIVE::template is_archivable<T>, void>
     {
       if(archive.Output())
         archive << size;
@@ -769,6 +854,9 @@ namespace ngcore
     NETGEN_INLINE void NothingToDelete () 
     { 
       mem_to_delete = nullptr;
+
+      // this memory is not managed by the Array anymore, so set the memory usage to 0
+      mt.Free(sizeof(T)*allocsize);
     }
 
     /// Change logical size. If necessary, do reallocation. Keeps contents.
@@ -801,6 +889,8 @@ namespace ngcore
     /// assigns memory from local heap
     NETGEN_INLINE const Array & Assign (size_t asize, LocalHeap & lh)
     {
+      if(mem_to_delete)
+        mt.Free(sizeof(T)*allocsize);
       delete [] mem_to_delete;
       size = allocsize = asize;
       data = lh.Alloc<T> (asize);
@@ -858,7 +948,7 @@ namespace ngcore
       size++;
     }
     
-    NETGEN_INLINE Array<T> & operator += (const T & el)
+    NETGEN_INLINE Array & operator += (const T & el)
     {
       Append (el);
       return *this;
@@ -880,7 +970,7 @@ namespace ngcore
 
 
     /// Delete element i. Move last element to position i.
-    NETGEN_INLINE void DeleteElement (size_t i)
+    NETGEN_INLINE void DeleteElement (IndexType i)
     {
       NETGEN_CHECK_RANGE(i,BASE,BASE+size);
       data[i-BASE] = std::move(data[size-1]);
@@ -889,10 +979,10 @@ namespace ngcore
 
 
     /// Delete element i. Move all remaining elements forward
-    NETGEN_INLINE void RemoveElement (size_t i)
+    NETGEN_INLINE void RemoveElement (IndexType i)
     {
       NETGEN_CHECK_RANGE(i, BASE, BASE+size);
-      for(size_t j = i; j < this->size-1; j++)
+      for(size_t j = i-BASE; j+1 < this->size; j++)
 	this->data[j] = this->data[j+1];
       this->size--;
     }
@@ -908,6 +998,8 @@ namespace ngcore
     /// Deallocate memory
     NETGEN_INLINE void DeleteAll ()
     {
+      if(mem_to_delete)
+        mt.Free(sizeof(T)*allocsize);
       delete [] mem_to_delete;
       mem_to_delete = NULL;
       data = 0;
@@ -924,16 +1016,25 @@ namespace ngcore
     /// array copy
     NETGEN_INLINE Array & operator= (const Array & a2)
     {
-      SetSize0 ();
-      SetSize (a2.Size());
-      for (size_t i = 0; i < size; i++)
-        data[i] = a2.data[i];
-      return *this;
+      if constexpr (std::is_copy_assignable<T>::value)
+        {
+          SetSize0 ();
+          SetSize (a2.Size());
+          for (size_t i = 0; i < size; i++)
+            data[i] = a2.data[i];
+          return *this;
+        }
+#ifndef __CUDA_ARCH__      
+      else
+        throw Exception(std::string("cannot copy Array of type ") + typeid(T).name());
+#endif
     }
 
+    
     /// steal array 
     NETGEN_INLINE Array & operator= (Array && a2)
     {
+      mt = std::move(a2.mt);
       ngcore::Swap (size, a2.size);
       ngcore::Swap (data, a2.data);
       ngcore::Swap (allocsize, a2.allocsize);
@@ -1007,12 +1108,21 @@ namespace ngcore
     
     NETGEN_INLINE void Swap (Array & b)
     {
+      mt = std::move(b.mt);
       ngcore::Swap (size, b.size);
       ngcore::Swap (data, b.data);
       ngcore::Swap (allocsize, b.allocsize);
       ngcore::Swap (mem_to_delete, b.mem_to_delete);
     }
 
+    NETGEN_INLINE void StartMemoryTracing () const
+    {
+      if(mem_to_delete)
+        mt.Alloc(sizeof(T) * allocsize);
+    }
+
+    const MemoryTracer& GetMemoryTracer() const { return mt; }
+
   private:
 
     /// resize array, at least to size minsize. copy contents
@@ -1029,6 +1139,7 @@ namespace ngcore
     
     T * hdata = data;
     data = new T[nsize];
+    mt.Alloc(sizeof(T) * nsize);
 
     if (hdata)
       {
@@ -1041,6 +1152,8 @@ namespace ngcore
         else
           for (size_t i = 0; i < mins; i++) data[i] = std::move(hdata[i]);
 #endif
+        if(mem_to_delete)
+          mt.Free(sizeof(T) * allocsize);
         delete [] mem_to_delete;
       }
 
@@ -1066,6 +1179,7 @@ namespace ngcore
     using Array<T>::allocsize;
     using Array<T>::data;
     using Array<T>::mem_to_delete;
+    using Array<T>::mt;
     // using Array<T>::ownmem;
 
   public:
@@ -1079,6 +1193,7 @@ namespace ngcore
           data = new T[asize];
           allocsize = size;
           mem_to_delete = data;
+          mt.Alloc(sizeof(T)*asize);
         }
     }
 
@@ -1099,6 +1214,7 @@ namespace ngcore
     ArrayMem(ArrayMem && a2)
       : Array<T> (a2.Size(), (T*)mem)
     {
+      mt = std::move(a2.mt);
       if (a2.mem_to_delete)
         {
           mem_to_delete = a2.mem_to_delete;
@@ -1124,6 +1240,14 @@ namespace ngcore
         data[cnt++] = val;
     }
   
+    template <typename T2>
+    ArrayMem (const BaseArrayObject<T2> & a2)
+      : ArrayMem (a2.Size())
+    {
+      for (size_t i : ngcore::Range(size))
+        data[i] = a2[i];
+    }
+
     
     ArrayMem & operator= (const T & val)
     {
@@ -1133,6 +1257,7 @@ namespace ngcore
 
     ArrayMem & operator= (ArrayMem && a2)
     {
+      mt = std::move(a2.mt);
       ngcore::Swap (mem_to_delete, a2.mem_to_delete);
       ngcore::Swap (allocsize, a2.allocsize);
       ngcore::Swap (size, a2.size);
@@ -1179,7 +1304,7 @@ namespace ngcore
 
 
   template <typename ... ARGS>
-  size_t ArraySize (Tuple<ARGS...> tup)  
+  size_t ArraySize (Tuple<ARGS...> /* tup */)  
   { return 0;}
   
   template <typename ... ARGS>
@@ -1192,7 +1317,7 @@ namespace ngcore
 
   
   template <typename T, typename ... ARGS>
-  void StoreToArray (FlatArray<T> a, Tuple<ARGS...> tup) { ; }
+  void StoreToArray (FlatArray<T> /* a */, Tuple<ARGS...> /* tup */) { ; }
   
   template <typename T, typename ... ARGS>
   void StoreToArray (FlatArray<T> a, Tuple<int,ARGS...> tup)
@@ -1265,6 +1390,12 @@ namespace ngcore
     return array;
   }
 
+  template <typename T, typename T2>
+  inline Array<T> operator+= (Array<T> && array, const BaseArrayObject<T2> & a2)
+  {
+    array += a2;
+    return std::move(array);
+  }
 
 
   /// bubble sort array
@@ -1422,6 +1553,8 @@ namespace ngcore
   }
                                
 
+  struct HTAHelp { };
+  
   // head-tail array
   template <size_t S, typename T>
   class HTArray
@@ -1429,8 +1562,23 @@ namespace ngcore
     HTArray<S-1,T> tail;
     T head;
   public:
-    HTArray () = default;
-    HTArray (const HTArray &) = default;
+    constexpr HTArray () = default;
+    constexpr HTArray (const HTArray &) = default;
+    template <typename T2>
+    constexpr HTArray (const HTArray<S,T2> & a2) : tail(a2.Tail()), head(a2.Head()) { ; }
+
+    constexpr HTArray (T v) : tail(v), head(v) { } // all the same
+    
+    template <class... T2,
+              std::enable_if_t<S==1+sizeof...(T2),bool> = true>
+    constexpr HTArray (const T &v, T2... rest)
+      : tail{HTAHelp(), v,rest...}, head(std::get<S-2>(std::tuple(rest...))) { }
+
+    template <class... T2>
+    constexpr HTArray (HTAHelp h, const T &v, T2... rest)
+      : tail{h, v,rest...}, head(std::get<S-2>(std::tuple(rest...))) { }
+
+    
     HTArray & operator= (const HTArray &) = default;
 
     T * Ptr () { return tail.Ptr(); }
@@ -1440,6 +1588,9 @@ namespace ngcore
     const T & operator[] (size_t i) const { return Ptr()[i]; }
     template <int NR>
     T & Elem() { return (NR==S-1) ? head : tail.template Elem<NR>(); }
+
+    auto Tail() const { return tail; }
+    auto Head() const { return head; }
   };
 
   template <typename T>
@@ -1447,8 +1598,16 @@ namespace ngcore
   {
     T head;
   public:
-    HTArray () = default;
-    HTArray (const HTArray &) = default;
+    constexpr HTArray () = default;
+    constexpr HTArray (const HTArray &) = default;
+    template <typename T2>
+    constexpr HTArray (const HTArray<1,T2> & a2) : head(a2.Head()) { ; }
+    constexpr HTArray (T v) : head(v) { } // all the same
+    template <class... T2>    
+    constexpr HTArray (HTAHelp h, const T &v, T2... rest)
+      : head(v) { } 
+
+    
     HTArray & operator= (const HTArray &) = default;
 
     T * Ptr () { return &head; }
@@ -1462,6 +1621,8 @@ namespace ngcore
       // assert(NR==0, "HTArray index error");
       return head;
     }
+
+    auto Head() const { return head; }
   };
 
   template <typename T>
@@ -1471,6 +1632,9 @@ namespace ngcore
   public:
     HTArray () = default;
     HTArray (const HTArray &) = default;
+    template <typename T2>
+    HTArray (const HTArray<0,T2> & a2) { ; }
+    constexpr HTArray (T v) { } // all the same    
     HTArray & operator= (const HTArray &) = default;
 
     /*
@@ -1510,6 +1674,64 @@ namespace ngcore
   {
     return ar.Ptr()+i;
   }
+
+
+
+
+
+
+
+
+
+
+  template <typename TIA, typename TIB>
+  class IteratorPair
+  {
+    TIA a;
+    TIB b;
+  public:
+    IteratorPair (TIA _a, TIB _b) : a(_a), b(_b) { ; }
+    
+    IteratorPair & operator++() { ++a; ++b; return *this; }
+    bool operator!= (const IteratorPair & it2)  { return a != it2.a; }
+    
+    auto operator*()
+    {
+      // return pair(*a,*b);
+      return std::pair<decltype(*a), decltype(*b)> (*a, *b); // keep reference
+    }
+  };
+  
+  
+  template <typename TA, typename TB>
+  class Zip
+  {
+    const TA & a;
+    const TB & b;
+  public:
+    Zip(const TA & _a, const TB & _b) : a(_a), b(_b) { ; }
+    auto begin() const { return IteratorPair(a.begin(), b.begin()); }
+    auto end() const { return IteratorPair(a.end(), b.end()); }
+  };
+
+  template <typename T>
+  inline size_t size (const BaseArrayObject<T> & ao) { return ao.Size(); }
+  
+  template <typename TA>
+  class Enumerate
+  {
+    IntRange r;
+    const TA & a;
+  public:
+    Enumerate(const TA & _a) : r(size(_a)), a(_a) { ; }
+    auto begin() const { return IteratorPair(r.begin(), a.begin()); }
+    auto end() const { return IteratorPair(r.end(), a.end()); }
+  };
+  
+  
+
+
+  
 }
 
 

libsrc/core/autodiffdiff.hpp

@@ -0,0 +1,733 @@
+#ifndef FILE_AUTODIFFDIFF
+#define FILE_AUTODIFFDIFF
+
+/**************************************************************************/
+/* File:   autodiffdiff.hpp                                               */
+/* Author: Joachim Schoeberl                                              */
+/* Date:   13. June. 05                                                   */
+/**************************************************************************/
+
+namespace ngcore
+{
+  using ngcore::IfPos;
+
+// Automatic second differentiation datatype
+
+
+/**
+   Datatype for automatic differentiation.  Contains function value,
+   D first derivatives, and D*D second derivatives. Algebraic operations are
+   overloaded by using product-rule etc. etc.
+**/
+template <int D, typename SCAL = double>
+class AutoDiffDiff
+{
+  SCAL val;
+  SCAL dval[D?D:1];
+  SCAL ddval[D?D*D:1];
+public:
+
+  typedef AutoDiffDiff<D, SCAL> TELEM;
+
+
+  /// elements are undefined
+  AutoDiffDiff  () throw() { ; }
+
+  /// copy constructor
+  AutoDiffDiff  (const AutoDiffDiff & ad2) throw()
+  {
+    val = ad2.val;
+    for (int i = 0; i < D; i++)
+      dval[i] = ad2.dval[i];
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = ad2.ddval[i];
+  }
+
+  /// initial object with constant value
+  AutoDiffDiff  (SCAL aval) throw()
+  {
+    val = aval;
+    for (int i = 0; i < D; i++)
+      dval[i] = 0;
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = 0;
+  }
+
+  /// initial object with value and derivative
+  AutoDiffDiff  (const AutoDiff<D, SCAL> & ad2) throw()
+  {
+    val = ad2.Value();
+    for (int i = 0; i < D; i++)
+      dval[i] = ad2.DValue(i);
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = 0;
+  }
+
+  /// init object with (val, e_diffindex)
+  AutoDiffDiff  (SCAL aval, int diffindex)  throw()
+  {
+    val = aval;
+    for (int i = 0; i < D; i++)
+      dval[i] = 0;
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = 0;
+    dval[diffindex] = 1;
+  }
+
+  NETGEN_INLINE AutoDiffDiff (SCAL aval, const SCAL * grad)
+  {
+    val = aval;
+    LoadGradient (grad);
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = 0;
+  }
+
+  NETGEN_INLINE AutoDiffDiff (SCAL aval, const SCAL * grad, const SCAL * hesse)
+  {
+    val = aval;
+    LoadGradient (grad);
+    LoadHessian (hesse);
+  }
+
+  /// assign constant value
+  AutoDiffDiff & operator= (SCAL aval) throw()
+  {
+    val = aval;
+    for (int i = 0; i < D; i++)
+      dval[i] = 0;
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = 0;
+    return *this;
+  }
+
+  NETGEN_INLINE void StoreGradient (SCAL * p) const 
+  {
+    for (int i = 0; i < D; i++)
+      p[i] = dval[i];
+  }
+
+  NETGEN_INLINE void LoadGradient (const SCAL * p) 
+  {
+    for (int i = 0; i < D; i++)
+      dval[i] = p[i];
+  }
+
+  NETGEN_INLINE void StoreHessian (SCAL * p) const 
+  {
+    for (int i = 0; i < D*D; i++)
+      p[i] = ddval[i];
+  }
+
+  NETGEN_INLINE void LoadHessian (const SCAL * p) 
+  {
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = p[i];
+  }
+
+  /// returns value
+  SCAL Value() const throw() { return val; }
+
+  /// returns partial derivative
+  SCAL DValue (int i) const throw() { return dval[i]; }
+
+  AutoDiff<D,SCAL> DValueAD (int i) const
+  {
+    AutoDiff<D,SCAL> r(dval[i]);
+    for (int j = 0; j < D; j++)
+      r.DValue(j) = ddval[i*D+j];
+    return r;
+  }
+  
+  /// returns partial derivative
+  SCAL DDValue (int i) const throw() { return ddval[i]; }
+
+  /// returns partial derivative
+  SCAL DDValue (int i, int j) const throw() { return ddval[i*D+j]; }
+
+  /// access value
+  SCAL & Value() throw() { return val; }
+
+  /// accesses partial derivative 
+  SCAL & DValue (int i) throw() { return dval[i]; }
+
+  /// accesses partial derivative 
+  SCAL & DDValue (int i) throw() { return ddval[i]; }
+
+  /// accesses partial derivative 
+  SCAL & DDValue (int i, int j) throw() { return ddval[i*D+j]; }
+
+  explicit operator AutoDiff<D,SCAL> () const
+  { return AutoDiff<D,SCAL> (val, &dval[0]); }
+  
+  /// add autodiffdiff object
+  AutoDiffDiff<D, SCAL> & operator+= (const AutoDiffDiff<D, SCAL> & y) throw()
+  {
+    val += y.val;
+    for (int i = 0; i < D; i++)
+      dval[i] += y.dval[i];
+    for (int i = 0; i < D*D; i++)
+      ddval[i] += y.ddval[i];
+    return *this;
+  }
+
+  /// subtract autodiffdiff object
+  AutoDiffDiff<D, SCAL> & operator-= (const AutoDiffDiff<D, SCAL> & y) throw()
+  {
+    val -= y.val;
+    for (int i = 0; i < D; i++)
+      dval[i] -= y.dval[i];
+    for (int i = 0; i < D*D; i++)
+      ddval[i] -= y.ddval[i];
+    return *this;
+  }
+
+  /// multiply with autodiffdiff object
+  AutoDiffDiff<D, SCAL> & operator*= (const AutoDiffDiff<D, SCAL> & y) throw()
+  {
+    for (int i = 0; i < D*D; i++)
+      ddval[i] = val * y.ddval[i] + y.val * ddval[i];
+
+    for (int i = 0; i < D; i++)
+      for (int j = 0; j < D; j++)
+	ddval[i*D+j] += dval[i] * y.dval[j] + dval[j] * y.dval[i];
+
+    for (int i = 0; i < D; i++)
+      {
+	dval[i] *= y.val;
+	dval[i] += val * y.dval[i];
+      }
+    val *= y.val;
+    return *this;
+  }
+
+  /// multiply with scalar
+  AutoDiffDiff<D, SCAL> & operator*= (const SCAL & y) throw()
+  {
+    for ( int i = 0; i < D*D; i++ )
+      ddval[i] *= y;
+    for (int i = 0; i < D; i++)
+      dval[i] *= y;
+    val *= y;
+    return *this;
+  }
+
+  /// divide by scalar
+  AutoDiffDiff<D, SCAL> & operator/= (const SCAL & y) throw()
+  {
+    SCAL iy = 1.0 / y;
+    for ( int i = 0; i < D*D; i++ )
+      ddval[i] *= iy;
+    for (int i = 0; i < D; i++)
+      dval[i] *= iy;
+    val *= iy;
+    return *this;
+  }
+
+  /// same value
+  bool operator== (SCAL val2) throw()
+  {
+    return val == val2;
+  }
+
+  /// different values 
+  bool operator!= (SCAL val2) throw()
+  {
+    return val != val2;
+  }
+
+  /// less 
+  bool operator< (SCAL val2) throw()
+  {
+    return val < val2;
+  }
+  
+  /// greater
+  bool operator> (SCAL val2) throw()
+  {
+    return val > val2;
+  }
+};
+
+
+//@{  AutoDiff helper functions.
+
+/// Prints AudoDiffDiff
+template<int D, typename SCAL>
+inline ostream & operator<< (ostream & ost, const AutoDiffDiff<D, SCAL> & x)
+{
+  ost << x.Value() << ", D = ";
+  for (int i = 0; i < D; i++)
+    ost << x.DValue(i) << " ";
+  ost << ", DD = ";
+  for (int i = 0; i < D*D; i++)
+    ost << x.DDValue(i) << " ";
+  return ost;
+}
+
+///
+template<int D, typename SCAL>
+inline AutoDiffDiff<D, SCAL> operator+ (const AutoDiffDiff<D, SCAL> & x, const AutoDiffDiff<D, SCAL> & y) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value () = x.Value()+y.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = x.DValue(i) + y.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = x.DDValue(i) + y.DDValue(i);
+  return res;
+}
+
+
+/// 
+template<int D, typename SCAL>
+inline AutoDiffDiff<D, SCAL> operator- (const AutoDiffDiff<D, SCAL> & x, const AutoDiffDiff<D, SCAL> & y) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = x.Value()-y.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = x.DValue(i) - y.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = x.DDValue(i) - y.DDValue(i);
+  return res;
+}
+
+
+///
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator+ (SCAL2 x, const AutoDiffDiff<D, SCAL> & y) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = x+y.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = y.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = y.DDValue(i);
+  return res;
+}
+
+///
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator+ (const AutoDiffDiff<D, SCAL> & y, SCAL2 x) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = x+y.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = y.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = y.DDValue(i);
+  return res;
+}
+
+
+///
+template<int D, typename SCAL>
+inline AutoDiffDiff<D, SCAL> operator- (const AutoDiffDiff<D, SCAL> & x) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = -x.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = -x.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = -x.DDValue(i);
+  return res;
+}
+
+///
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator- (const AutoDiffDiff<D, SCAL> & x, SCAL2 y) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = x.Value()-y;
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = x.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = x.DDValue(i);
+  return res;
+}
+
+///
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator- (SCAL2 x, const AutoDiffDiff<D, SCAL> & y) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = x-y.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = -y.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = -y.DDValue(i);
+  return res;
+}
+
+
+///
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator* (SCAL2 x, const AutoDiffDiff<D, SCAL> & y) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = x*y.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = x*y.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = x*y.DDValue(i);
+  return res;
+}
+
+///
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator* (const AutoDiffDiff<D, SCAL> & y, SCAL2 x) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = x*y.Value();
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = x*y.DValue(i);
+  for (int i = 0; i < D*D; i++)
+    res.DDValue(i) = x*y.DDValue(i);
+  return res;
+}
+
+///
+template<int D, typename SCAL>
+inline AutoDiffDiff<D, SCAL> operator* (const AutoDiffDiff<D, SCAL> & x, const AutoDiffDiff<D, SCAL> & y) throw()
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL hx = x.Value();
+  SCAL hy = y.Value();
+
+  res.Value() = hx*hy;
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = hx*y.DValue(i) + hy*x.DValue(i);
+
+  for (int i = 0; i < D; i++)
+    for (int j = 0; j < D; j++)
+      res.DDValue(i,j) = hx * y.DDValue(i,j) + hy * x.DDValue(i,j)
+	+ x.DValue(i) * y.DValue(j) + x.DValue(j) * y.DValue(i);
+
+  return res;
+}
+
+
+
+template<int D, typename SCAL>
+inline AutoDiffDiff<D, SCAL> Inv (const AutoDiffDiff<D, SCAL> & x)
+{
+  AutoDiffDiff<D, SCAL> res(1.0 / x.Value());
+  for (int i = 0; i < D; i++)
+    res.DValue(i) = -x.DValue(i) / (x.Value() * x.Value());
+
+  SCAL fac1 = 2/(x.Value()*x.Value()*x.Value());
+  SCAL fac2 = 1/sqr(x.Value());
+  for (int i = 0; i < D; i++)
+    for (int j = 0; j < D; j++)
+      res.DDValue(i,j) = fac1*x.DValue(i)*x.DValue(j) - fac2*x.DDValue(i,j);
+  return res;
+}
+
+
+template<int D, typename SCAL>
+inline AutoDiffDiff<D, SCAL> operator/ (const AutoDiffDiff<D, SCAL> & x, const AutoDiffDiff<D, SCAL> & y)
+{
+  return x * Inv (y);
+}
+
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator/ (const AutoDiffDiff<D, SCAL> & x, SCAL2 y)
+{
+  return (1/y) * x;
+}
+
+template<int D, typename SCAL, typename SCAL2,
+           typename std::enable_if<std::is_convertible<SCAL2,SCAL>::value, int>::type = 0>
+inline AutoDiffDiff<D, SCAL> operator/ (SCAL2 x, const AutoDiffDiff<D, SCAL> & y)
+{
+  return x * Inv(y);
+}
+
+
+template<int D, typename SCAL>
+inline AutoDiffDiff<D, SCAL> sqrt (const AutoDiffDiff<D, SCAL> & x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = sqrt(x.Value());
+  for (int j = 0; j < D; j++)
+    res.DValue(j) = IfZero(x.DValue(j),SCAL{0.},0.5 / res.Value() * x.DValue(j));
+
+  
+  for (int i = 0; i < D; i++)
+    for (int j = 0; j < D; j++)
+      res.DDValue(i,j) = IfZero(x.DDValue(i,j)+x.DValue(i) * x.DValue(j),SCAL{0.},0.5/res.Value() * x.DDValue(i,j) - 0.25 / (x.Value()*res.Value()) * x.DValue(i) * x.DValue(j));
+
+  return res;
+}
+
+// df(u)/dx  = exp(x) * du/dx
+// d^2 f(u) / dx^2 = exp(x) * (du/dx)^2 + exp(x) * d^2u /dx^2
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> exp (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = exp(x.Value());
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = x.DValue(k) * res.Value();
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = (x.DValue(k) * x.DValue(l)+x.DDValue(k,l)) * res.Value();
+  return res;
+}
+
+using std::pow;
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D,SCAL> pow (AutoDiffDiff<D,SCAL> x, AutoDiffDiff<D,SCAL> y )
+{
+  return exp(log(x)*y);
+}
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> log (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  res.Value() = log(x.Value());
+  SCAL xinv = 1.0/x.Value();
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = x.DValue(k) * xinv;
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = -xinv*xinv*x.DValue(k) * x.DValue(l) + xinv * x.DDValue(k,l);
+  return res;
+}
+
+
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> sin (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL s = sin(x.Value());
+  SCAL c = cos(x.Value());
+  
+  res.Value() = s;
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = x.DValue(k) * c;
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = -s * x.DValue(k) * x.DValue(l) + c * x.DDValue(k,l);
+  return res;
+}
+
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> cos (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL s = sin(x.Value());
+  SCAL c = cos(x.Value());
+  
+  res.Value() = c;
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = -s * x.DValue(k);
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = -c * x.DValue(k) * x.DValue(l) - s * x.DDValue(k,l);
+  return res;
+}
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> tan (AutoDiffDiff<D, SCAL> x)
+{ return sin(x) / cos(x); }
+
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> atan (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL a = atan(x.Value());
+  res.Value() = a;
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = x.DValue(k)/(1+x.Value()*x.Value()) ;
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = -2*x.Value()/((1+x.Value()*x.Value())*(1+x.Value()*x.Value())) * x.DValue(k) * x.DValue(l) + x.DDValue(k,l)/(1+x.Value()*x.Value());
+  return res;
+}
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> atan2 (AutoDiffDiff<D, SCAL> x,AutoDiffDiff<D, SCAL> y)
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL a = atan2(x.Value(), y.Value());
+  res.Value() = a;
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = (x.Value()*y.DValue(k)-y.Value()*x.DValue(k))/(y.Value()*y.Value()+x.Value()*x.Value());
+
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = (x.DValue(k)*y.DValue(l)+x.Value()*y.DDValue(l,k) - y.DValue(k)*x.DValue(l) - y.Value()*x.DDValue(l,k))/(y.Value()*y.Value()+x.Value()*x.Value()) - 2 * (x.Value()*y.DValue(k)-y.Value()*x.DValue(k)) * (x.Value()*x.DValue(k) + y.Value()*y.DValue(k))/( (y.Value()*y.Value()+x.Value()*x.Value()) * (y.Value()*y.Value()+x.Value()*x.Value()) );
+  return res;
+}
+
+
+
+using std::acos;
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D,SCAL> acos (AutoDiffDiff<D,SCAL> x)
+{
+  AutoDiffDiff<D,SCAL> res;
+  SCAL a = acos(x.Value());
+  res.Value() = a;
+  auto omaa = 1-x.Value()*x.Value();
+  auto s = sqrt(omaa);
+  SCAL da = -1 / s;
+  SCAL dda = -x.Value() / (s*omaa);
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = x.DValue(k)*da;
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = dda * x.DValue(k) * x.DValue(l) + da * x.DDValue(k,l);
+  
+  return res;
+}
+
+
+using std::acos;
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D,SCAL> asin (AutoDiffDiff<D,SCAL> x)
+{
+  AutoDiffDiff<D,SCAL> res;
+  SCAL a = asin(x.Value());
+  res.Value() = a;
+  auto omaa = 1-x.Value()*x.Value();
+  auto s = sqrt(omaa);
+  SCAL da = 1 / s;
+  SCAL dda = x.Value() / (s*omaa);
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = x.DValue(k)*da;
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = dda * x.DValue(k) * x.DValue(l) + da * x.DDValue(k,l);
+  
+  return res;
+}
+
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> sinh (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL sh = sinh(x.Value());
+  SCAL ch = cosh(x.Value());
+  
+  res.Value() = sh;
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = x.DValue(k) * ch;
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = sh * x.DValue(k) * x.DValue(l) + ch * x.DDValue(k,l);
+  return res;
+}
+
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> cosh (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL sh = sinh(x.Value());
+  SCAL ch = cosh(x.Value());
+  
+  res.Value() = ch;
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = sh * x.DValue(k);
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = ch * x.DValue(k) * x.DValue(l) + sh * x.DDValue(k,l);
+  return res;
+}
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D, SCAL> erf (AutoDiffDiff<D, SCAL> x)
+{
+  AutoDiffDiff<D, SCAL> res;
+  SCAL derf = 2. / sqrt(M_PI) * exp(- x.Value() * x.Value());
+  
+  res.Value() = erf(x.Value());
+  for (int k = 0; k < D; k++)
+    res.DValue(k) = - derf * x.DValue(k);
+  for (int k = 0; k < D; k++)
+    for (int l = 0; l < D; l++)
+      res.DDValue(k,l) = derf * (x.DDValue(k, l) - 2 * x.Value() * x.DValue(k) * x.DValue(l));
+  return res;
+}
+
+using std::floor;
+template<int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D,SCAL> floor (const AutoDiffDiff<D,SCAL> & x)
+{
+  return floor(x.Value());
+}
+
+using std::ceil;
+template<int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D,SCAL> ceil (const AutoDiffDiff<D,SCAL> & x)
+{
+  return ceil(x.Value());  
+}
+
+
+template <int D, typename SCAL, typename TB, typename TC>
+auto IfPos (AutoDiffDiff<D,SCAL> a, TB b, TC c) -> decltype(IfPos (a.Value(), b, c))
+{
+  return IfPos (a.Value(), b, c);
+}
+
+template <int D, typename SCAL>
+NETGEN_INLINE AutoDiffDiff<D,SCAL> IfPos (SCAL /* SIMD<double> */ a, AutoDiffDiff<D,SCAL> b, AutoDiffDiff<D,SCAL> c)
+{
+  AutoDiffDiff<D,SCAL> res;
+  res.Value() = IfPos (a, b.Value(), c.Value());
+  for (int j = 0; j < D; j++)
+  {
+    res.DValue(j) = IfPos (a, b.DValue(j), c.DValue(j));
+    res.DDValue(j) = IfPos (a, b.DDValue(j), c.DDValue(j));
+  }
+  return res;
+}
+
+template <int D, typename SCAL, typename TC>
+NETGEN_INLINE AutoDiffDiff<D,SCAL> IfPos (SCAL /* SIMD<double> */ a, AutoDiffDiff<D,SCAL> b, TC c)
+{
+  return IfPos (a, b, AutoDiffDiff<D,SCAL> (c));
+}
+
+
+
+
+//@}
+
+}
+
+
+namespace ngbla
+{
+  template <typename T> struct is_scalar_type;
+  template <int D, typename T>
+  struct is_scalar_type<ngcore::AutoDiffDiff<D,T>> { static constexpr bool value = true; };
+
+
+  // not meaningful for AutoDiff<D,Complex>, since this is
+  // not (complex) differentiable anyway
+  template<int D, typename SCAL>
+  inline auto L2Norm2 (const ngcore::AutoDiffDiff<D,SCAL> & x) 
+  {
+    return x*x;
+  }
+  
+}
+
+
+
+#endif

libsrc/core/bitarray.cpp

@@ -9,6 +9,7 @@
 */
 
 #include "bitarray.hpp"
+#include "archive.hpp"
 
 namespace ngcore
 {
@@ -36,10 +37,16 @@ namespace ngcore
   void BitArray :: SetSize (size_t asize)
   {
     if (size == asize) return;
-    if (owns_data) delete [] data;
+    if (owns_data)
+      {
+        delete [] data;
+        mt.Free(GetMemoryUsage());
+      }
 
     size = asize;
     data = new unsigned char [Addr (size)+1];
+    owns_data = true;
+    mt.Alloc(GetMemoryUsage());
   }
 
   BitArray & BitArray :: Set () throw()
@@ -83,6 +90,18 @@ namespace ngcore
     return *this;
   }
 
+  bool BitArray :: operator==(const BitArray& other) const
+  {
+    if(size != other.Size())
+      return false;
+    for(auto i : Range(size/CHAR_BIT))
+      if(data[i] != other.data[i])
+        return false;
+    for(auto i : Range(size%CHAR_BIT))
+      if(Test(i + CHAR_BIT * (size/CHAR_BIT)) != other.Test(i + CHAR_BIT * (size/CHAR_BIT)))
+        return false;
+    return true;
+  }
 
   BitArray & BitArray :: operator= (const BitArray & ba2)
   {
@@ -115,29 +134,52 @@ namespace ngcore
     return cnt;
   }
 
-  Archive & operator & (Archive & archive, BitArray & ba)
+  void BitArray :: DoArchive(Archive& archive)
   {
-    if (archive.Output())
+    if(archive.GetVersion("netgen") >= "v6.2.2007-62")
       {
-        archive << ba.Size();
-        for (size_t i = 0; i < ba.Size(); i++)
-          archive << ba[i];
+        archive.NeedsVersion("netgen", "v6.2.2007-62");
+        auto size = Size();
+        archive & size;
+        if(archive.Input())
+          SetSize(size);
+        if(archive.GetVersion("netgen") < "v6.2.2009-20")
+          archive.Do(data, size/CHAR_BIT+1);
+        else
+          {
+            archive.NeedsVersion("netgen", "v6.2.2009-20");
+            archive.Do(data, size/CHAR_BIT);
+            for(size_t i = 0; i < size%CHAR_BIT; i++)
+              {
+                size_t index =  CHAR_BIT * (size/CHAR_BIT) + i;
+                bool b = Test(index);
+                archive & b;
+                b ? SetBit(index) : Clear(index);
+              }
+          }
       }
     else
       {
-        size_t size;
-        archive & size;
-        ba.SetSize (size);
-        ba.Clear();
-        for (size_t i = 0; i < size; i++)
+        if (archive.Output())
           {
-            bool b;
-            archive & b;
-            if (b) ba.SetBit(i);
+            throw Exception("should not get here");
+            archive << Size();
+            for (size_t i = 0; i < Size(); i++)
+              archive << (*this)[i];
+          }
+        else
+          {
+            size_t size;
+            archive & size;
+            SetSize (size);
+            Clear();
+            for (size_t i = 0; i < size; i++)
+              {
+                bool b;
+                archive & b;
+                if (b) SetBit(i);
+              }
           }
       }
-    return archive;
   }
-
-
-}
+} // namespace ngcore

libsrc/core/bitarray.hpp

@@ -11,7 +11,6 @@
 #include <cstring>
 #include <ostream>
 
-#include "archive.hpp"
 #include "array.hpp"
 #include "localheap.hpp"
 #include "ngcore_api.hpp"
@@ -50,6 +49,7 @@ public:
   {
     ba2.owns_data = false;
     ba2.data = nullptr;
+    mt = std::move(ba2.mt);
   }
 
   template <typename T>
@@ -60,13 +60,17 @@ public:
     int cnt = 0;
     for (auto i = list.begin(); i < list.end(); i++, cnt++)
       if (*i) SetBit(cnt);
+    StartMemoryTracing();
   }
 
   /// delete data
   ~BitArray ()
   {
     if (owns_data)
+    {
       delete [] data;
+      mt.Free(GetMemoryUsage());
+    }
   }
 
   /// Set size, loose values
@@ -131,6 +135,7 @@ public:
     return Test(i);
   }
 
+  NGCORE_API bool operator==(const BitArray& other) const;
 
   /// invert all bits
   NGCORE_API BitArray & Invert ();
@@ -145,6 +150,18 @@ public:
   NGCORE_API BitArray & operator= (const BitArray & ba2);
 
   NGCORE_API size_t NumSet () const;
+
+  NGCORE_API void DoArchive(class Archive& archive);
+  
+  NGCORE_API auto * Data() const { return data; }
+
+  const size_t GetMemoryUsage() const { return owns_data ? (size+CHAR_BIT-1)/CHAR_BIT : 0; }
+  const MemoryTracer& GetMemoryTracer() const { return mt; }
+  void StartMemoryTracing() const
+  {
+    mt.Alloc(GetMemoryUsage());
+  }
+
 private:
   ///
   unsigned char Mask (size_t i) const
@@ -154,6 +171,7 @@ private:
   size_t Addr (size_t i) const
   { return (i / CHAR_BIT); }
 
+  MemoryTracer mt;
 };
 
 
@@ -190,11 +208,33 @@ private:
     return res;
   }
 
-
   NGCORE_API std::ostream & operator<<(std::ostream & s, const BitArray & ba);
 
-  NGCORE_API Archive & operator & (Archive & archive, BitArray & ba);
+
+
+  template <typename IndexType>
+  class TBitArray : public BitArray
+  {
+  public:
+    using BitArray::BitArray;
+
+    void SetBit (IndexType i) { BitArray::SetBit(i-IndexBASE<IndexType>()); }
+    void Clear () { BitArray::Clear(); }
+    void Clear (IndexType i) { BitArray::Clear(i-IndexBASE<IndexType>()); }
+    void SetBitAtomic (IndexType i) { BitArray::SetBitAtomic(i-IndexBASE<IndexType>()); }
+    bool Test (IndexType i) const { return BitArray::Test(i-IndexBASE<IndexType>()); }
+    
+    bool operator[] (IndexType i) const { return Test(i); } 
+    T_Range<IndexType> Range() const { return { IndexBASE<IndexType>(), IndexBASE<IndexType>()+Size() }; }
+    NGCORE_API TBitArray & Or (const TBitArray & ba2)
+    {
+      BitArray::Or(ba2);
+      return *this;
+    }
+
+  };
 
 } // namespace ngcore
 
+  
 #endif // NETGEN_CORE_BITARRAY

libsrc/core/exception.cpp

@@ -1,15 +1,69 @@
 #include "exception.hpp"
 #include "utils.hpp"
 
+#ifdef EMSCRIPTEN
+#include <iostream>
+#endif // EMSCRIPTEN
+
 namespace ngcore
 {
   Exception :: Exception(const std::string& s)
-    : m_what(s) {}
+    : m_what(s) {
+  #ifdef EMSCRIPTEN
+  std::cout << "THROW Exception " << s << std::endl;
+  #endif
+}
   
   Exception :: Exception(const char* s)
-    : m_what(s) {}
+    : m_what(s) {
+  #ifdef EMSCRIPTEN
+  std::cout << "THROW Exception " << s << std::endl;
+  #endif
+
+}
 
 
+  Exception :: Exception(std::string_view s1, std::string_view s2)
+    : Exception(std::string(s1)+std::string(s2))
+  { }
+
+  Exception :: Exception(std::string_view s1, std::string_view s2, std::string_view s3)
+    : Exception(std::string(s1)+std::string(s2)+std::string(s3))
+  { }
+
+
+  void Exception :: Throw (std::string_view s1)
+  {
+    throw Exception(std::string(s1));
+  }
+
+  void Exception :: Throw (std::string_view s1, std::string_view s2)
+  {
+    throw Exception(std::string(s1)+std::string(s2));
+  }
+    
+  void Exception :: Throw (std::string_view s1, std::string_view s2, std::string_view s3)
+  {
+    throw Exception(std::string(s1)+std::string(s2)+std::string(s3));
+  }
+  
+  
+  RangeException :: RangeException (// const std::string & where,
+                                    const char * where,
+                                    ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax) : Exception("")
+  {
+    std::stringstream str;
+    str << where << ": index " << ind << " out of range [" << imin << "," << imax << ")\n";
+    Append (str.str());
+    Append (GetBackTrace());
+  }
+  
+
+  void ThrowRangeException(const char * s, ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax)
+  {
+    throw RangeException(s, ind, imin, imax);
+  }
+  
   void ThrowException(const std::string & s)
   {
     throw Exception (s);
@@ -19,11 +73,18 @@ namespace ngcore
   {
     throw Exception (s);
   }
+
+
+  void ThrowNotTheSameException(const char * s, ptrdiff_t a, ptrdiff_t b)
+  {
+    throw ngcore::Exception(std::string(s) + ", a="+ToString(a) + ", b="+ToString(b) + GetBackTrace());     
+  }
+  
 } // namespace ngcore
 
 
 // ********* STUFF FOR GETBACKTRACE ***************************
-#ifdef __GNUC__
+#if defined __GNUC__ && !defined __EMSCRIPTEN__
 
 #include <execinfo.h>
 #include <string.h>
@@ -67,7 +128,7 @@ namespace ngcore
       // 1   libngcore.dylib                     0x000000010ddb298c _ZL21ngcore_signal_handleri + 316
       constexpr char reset_shell[] = "\033[0m";
       constexpr char green[] = "\033[32m";
-      constexpr char yellow[] = "\033[33m";
+      [[maybe_unused]] constexpr char yellow[] = "\033[33m";
 
       std::istringstream in(s);
 
@@ -122,7 +183,7 @@ namespace ngcore
       auto libname = s.substr(0, brace_open_pos);
       auto funcname = s.substr(brace_open_pos+1, plus_pos - brace_open_pos - 1);
       auto offset = std::strtoul(s.substr(plus_pos+1, brace_close_pos - plus_pos - 1).c_str(), 0, 16);
-      auto position = std::strtoul(s.substr(bracket_open_pos+1, bracket_close_pos - bracket_open_pos - 1).c_str(), 0, 16);
+      // auto position = std::strtoul(s.substr(bracket_open_pos+1, bracket_close_pos - bracket_open_pos - 1).c_str(), 0, 16);
       std::stringstream out;
 
       if(!funcname.empty())
@@ -181,7 +242,7 @@ namespace ngcore
     for (i = 1; i < bt_size-1; i++)
       {
         dladdr(bt[i], &info);
-        size_t len = strlen(bt_syms[i]);
+        // size_t len = strlen(bt_syms[i]);
         result << '#'<< i << '\t' << detail::TranslateBacktrace( bt_syms[i], info.dli_fname );
       }
     free(bt_syms);
@@ -217,13 +278,16 @@ static void ngcore_signal_handler(int sig)
 // register signal handler when library is loaded
 static bool dummy = []()
 {
-    signal(SIGABRT, ngcore_signal_handler);
-    signal(SIGILL, ngcore_signal_handler);
-    signal(SIGSEGV, ngcore_signal_handler);
+    if(getenv("NG_BACKTRACE"))
+    {
+      signal(SIGABRT, ngcore_signal_handler);
+      signal(SIGILL, ngcore_signal_handler);
+      signal(SIGSEGV, ngcore_signal_handler);
+    }
     return true;
 }();
 
-#else // __GNUC__
+#else // __GNUC__ and not __EMSCRIPTEN__
 
 namespace ngcore
 {

libsrc/core/exception.hpp

@@ -1,16 +1,17 @@
 #ifndef NETGEN_CORE_EXCEPTION_HPP
 #define NETGEN_CORE_EXCEPTION_HPP
 
+#include <cstddef>
 #include <sstream>         // for stringstream
 #include <stdexcept>       // for exception
 #include <string>          // for string
 
 #include "ngcore_api.hpp"  // for NGCORE_API
+#include "utils.hpp"       // for ToString
 
 
 namespace ngcore
 {
-
   NGCORE_API std::string GetBackTrace();
 
   // Exception for code that shouldn't be executed
@@ -33,8 +34,14 @@ namespace ngcore
     Exception(Exception&&) = default;
     Exception(const std::string& s); //  : m_what(s) {}
     Exception(const char* s); //  : m_what(s) {}
+    Exception(std::string_view s1, std::string_view s2);
+    Exception(std::string_view s1, std::string_view s2, std::string_view s3);
     ~Exception() override = default;
 
+    [[noreturn]] static void Throw (std::string_view s1);
+    [[noreturn]] static void Throw (std::string_view s1, std::string_view s2);
+    [[noreturn]] static void Throw (std::string_view s1, std::string_view s2, std::string_view s3);
+    
     Exception& operator =(const Exception&) = default;
     Exception& operator =(Exception&&) noexcept = default;
 
@@ -50,23 +57,26 @@ namespace ngcore
     const char* what() const noexcept override { return m_what.c_str(); }
   };
   
-  NGCORE_API void ThrowException(const std::string & s);
-  NGCORE_API void ThrowException(const char * s);
+  [[noreturn]] NGCORE_API void ThrowException(const std::string & s);
+  [[noreturn]] NGCORE_API void ThrowException(const char * s);
   
   // Out of Range exception
   class NGCORE_API RangeException : public Exception
   {
   public:
     /// where it occurs, index, minimal and maximal indices
-    RangeException (const std::string & where,
-                    int ind, int imin, int imax) : Exception("")
+    RangeException (// const std::string & where,
+                    const char * where,
+                    ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax);
+    /*
+    : Exception("")
       {
         std::stringstream str;
         str << where << ": index " << ind << " out of range [" << imin << "," << imax << ")\n";
         Append (str.str());
         Append (GetBackTrace());
       }
-
+    */
     template<typename T>
     RangeException(const std::string& where, const T& value)
     {
@@ -76,9 +86,40 @@ namespace ngcore
     }
   };
 
+  [[noreturn]] NGCORE_API void ThrowRangeException(const char * s, ptrdiff_t ind, ptrdiff_t imin, ptrdiff_t imax);
+  [[noreturn]] NGCORE_API void ThrowNotTheSameException(const char * s, ptrdiff_t a, ptrdiff_t b);
+  
+  
   // Exception used if no simd implementation is available to fall back to standard evaluation
   class NGCORE_API ExceptionNOSIMD : public Exception
   { public: using Exception::Exception; };
+
+  template <typename T>
+  struct IsSafe {
+    constexpr operator bool() const { return false; } };
+
+  namespace detail {
+    template <typename T, typename Tmin, typename Tmax>
+    inline static constexpr void CheckRange(const char * s, const T& n, Tmin first, Tmax next)
+    {
+      if constexpr (!IsSafe<decltype(n)>())
+        if (n<first || n>=next)
+          ThrowRangeException(s, ptrdiff_t(n), ptrdiff_t(first), ptrdiff_t(next));
+    }
+
+    template <typename Ta, typename Tb>
+    inline static constexpr void CheckSame(const char * s, const Ta& a, const Tb& b)
+    {
+     if constexpr (!IsSafe<decltype(a)>() || !IsSafe<decltype(b)>())
+      if(a != b)
+      {
+        if constexpr(std::is_integral_v<decltype(a)> && std::is_same_v<decltype(a),decltype(b)>)
+          ThrowNotTheSameException(s, long(a), long(b)); \
+        else
+          throw Exception(std::string(s) + "\t: not the same, a="+ToString(a) + ", b="+ngcore::ToString(b) + GetBackTrace());
+      }
+    }
+  } // namespace detail
 } // namespace ngcore
 
 #define NETGEN_CORE_NGEXEPTION_STR_HELPER(x) #x
@@ -87,12 +128,18 @@ namespace ngcore
 // Convenience macro to append file name and line of exception origin to the string
 #define NG_EXCEPTION(s) ngcore::Exception(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t"+std::string(s))
 
-#ifdef NETGEN_ENABLE_CHECK_RANGE
-#define NETGEN_CHECK_RANGE(value, min, max_plus_one) \
-  { if ((value)<(min) ||  (value)>=(max_plus_one)) \
-      throw ngcore::RangeException(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t", (value), (min), (max_plus_one)); }
-#else // NETGEN_ENABLE_CHECK_RANGE
-#define NETGEN_CHECK_RANGE(value, min, max)
-#endif // NETGEN_ENABLE_CHECK_RANGE
+#if defined(NETGEN_ENABLE_CHECK_RANGE) && !defined(__CUDA_ARCH__)
+#define NETGEN_CHECK_RANGE(value, min, max_plus_one) ngcore::detail::CheckRange(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t", value, min, max_plus_one);
+#define NETGEN_CHECK_SAME(a,b) ngcore::detail::CheckSame(__FILE__ ":" NETGEN_CORE_NGEXEPTION_STR(__LINE__) "\t", a, b);
 
+#define NETGEN_NOEXCEPT 
+#else // defined(NETGEN_ENABLE_CHECK_RANGE) && !defined(__CUDA_ARCH__)
+#define NETGEN_CHECK_RANGE(value, min, max)
+#define NETGEN_CHECK_SAME(a,b)
+// #define NETGEN_CHECK_SHAPE(a,b)
+#define NETGEN_NOEXCEPT noexcept
+#endif // defined(NETGEN_ENABLE_CHECK_RANGE) && !defined(__CUDA_ARCH__)
+
+
+  
 #endif // NETGEN_CORE_EXCEPTION_HPP

libsrc/core/flags.cpp

@@ -4,6 +4,7 @@
 /* Date:   10. Oct. 96                                                    */
 /**************************************************************************/
 
+#include "archive.hpp"
 #include "flags.hpp"
 
 #ifdef WIN32
@@ -15,6 +16,7 @@
 namespace ngcore
 {
   using std::string;
+  using std::string_view;
   using std::endl;
   Flags :: Flags () { ; }
 
@@ -51,6 +53,10 @@ namespace ngcore
 	auto lflags = flags.GetFlagsFlag (i, name);
 	SetFlag (name, lflags);
       }
+    for(auto i : Range(flags.anyflags.Size()))
+      {
+        SetFlag(flags.anyflags.GetName(i), flags.anyflags[i]);
+      }
   }
   
   Flags :: Flags (Flags && flags)
@@ -178,7 +184,11 @@ namespace ngcore
     return *this;
   }
 
-
+  Flags & Flags :: SetFlag (const string & name, const std::any & val)
+  {
+    anyflags.Set(name, val);
+    return *this;
+  }
 
   string Flags :: GetStringFlag (const string & name, const char * def) const
   {
@@ -200,18 +210,18 @@ namespace ngcore
   }
 
 
-  double Flags :: GetNumFlag (const string & name, double def) const
+  double Flags :: GetNumFlag (string_view name, double def) const
   {
     if (numflags.Used (name))
-      return numflags[name];
+      return numflags[string(name)];
     else
       return def;
   }
   
-  const double * Flags :: GetNumFlagPtr (const string & name) const
+  const double * Flags :: GetNumFlagPtr (string_view name) const
   {
     if (numflags.Used (name))
-      return & ((SymbolTable<double>&)numflags)[name];
+      return & ((SymbolTable<double>&)numflags)[string(name)];
     else
       return NULL;
   }
@@ -230,16 +240,16 @@ namespace ngcore
     return defflags.Used (name);
   }
   */
-  bool Flags :: GetDefineFlag (const string & name) const throw()
+  bool Flags :: GetDefineFlag (string_view name) const throw()
   {
-    if (!defflags.Used (name)) return false;
-    return defflags[name];
+    if (!defflags.Used (string(name))) return false;
+    return defflags[string(name)];
   }
 
-  xbool Flags :: GetDefineFlagX (const string & name) const throw()
+  xbool Flags :: GetDefineFlagX (string_view name) const throw()
   {
-    if (!defflags.Used (name)) return maybe;
-    return bool(defflags[name]);
+    if (!defflags.Used (string(name))) return maybe;
+    return bool(defflags[string(name)]);
   }
 
 
@@ -279,31 +289,44 @@ namespace ngcore
       }
   }
 
-  bool Flags :: StringFlagDefined (const string & name) const
+  const std::any& Flags:: GetAnyFlag(const std::string& name) const
+  {
+    if(anyflags.Used(name))
+      return anyflags[name];
+    static std::any empty;
+    return empty;
+  }
+
+  bool Flags :: StringFlagDefined (string_view name) const noexcept
   {
     return strflags.Used (name);
   }
 
-  bool Flags :: NumFlagDefined (const string &name) const
+  bool Flags :: NumFlagDefined (string_view name) const noexcept
   {
     return numflags.Used (name);
   }
 
-  bool Flags :: FlagsFlagDefined (const string &name) const
+  bool Flags :: FlagsFlagDefined (string_view name) const noexcept
   {
     return flaglistflags.Used (name);
   }
   
-  bool Flags :: StringListFlagDefined (const string & name) const
+  bool Flags :: StringListFlagDefined (string_view name) const noexcept
   {
     return strlistflags.Used (name);
   }
 
-  bool Flags :: NumListFlagDefined (const string & name) const
+  bool Flags :: NumListFlagDefined (string_view name) const noexcept
   {
     return numlistflags.Used (name);
   }
 
+  bool Flags :: AnyFlagDefined (string_view name) const noexcept
+  {
+    return anyflags.Used(name);
+  }
+
   void Flags :: SaveFlags (ostream & str) const
   {
     for (int i = 0; i < strflags.Size(); i++)
@@ -584,7 +607,7 @@ namespace ngcore
 	      }
 	    else
 	      {
-                // to be cleand up ...
+                // to be cleaned up ...
 		Array<char *> strs;
 
 		posbrack++;

libsrc/core/flags.hpp

@@ -11,6 +11,7 @@
 #include <iostream>
 #include <memory>
 #include <string>
+#include <any>
 
 #include "array.hpp"
 #include "symboltable.hpp"
@@ -38,6 +39,8 @@ namespace ngcore
     SymbolTable<std::shared_ptr<Array<double>>> numlistflags;
     /// flags list flags
     SymbolTable<Flags> flaglistflags;
+    /// any object can be stored as a flag
+    SymbolTable<std::any> anyflags;
   public:
     /// no flags
     Flags ();
@@ -55,7 +58,7 @@ namespace ngcore
     Flags & operator= (const Flags & f2) = default;
     Flags & operator= (Flags && f2) = default;
 
-    void DoArchive(Archive& ar);
+    void DoArchive(class Archive& ar);
 
     void Update(const Flags& other);
   
@@ -94,6 +97,8 @@ namespace ngcore
     Flags &  SetFlag (const std::string & name, const Array<std::string> & val);
     /// Sets double array flag
     Flags &  SetFlag (const std::string & name, const Array<double> & val);
+    /// Sets any flag
+    Flags &  SetFlag(const std::string& name, const std::any& val);
 
 
     Flags SetFlag (const char * name, bool b = true) &&;
@@ -120,33 +125,35 @@ namespace ngcore
     /// Returns std::string flag, default value if not exists
     std::string GetStringFlag (const std::string & name, std::string def = "") const;
     /// Returns numerical flag, default value if not exists
-    double GetNumFlag (const std::string & name, double def) const;
+    double GetNumFlag (std::string_view name, double def) const;
     /// Returns address of numerical flag, null if not exists
-    const double * GetNumFlagPtr (const std::string & name) const;
+    const double * GetNumFlagPtr (std::string_view name) const;
     /// Returns address of numerical flag, null if not exists
     double * GetNumFlagPtr (const std::string & name);
     /// Returns boolean flag
     // int GetDefineFlag (const char * name) const;
-    bool GetDefineFlag (const std::string & name) const throw();
-    xbool GetDefineFlagX (const std::string & name) const throw();
+    bool GetDefineFlag (std::string_view name) const  noexcept;
+    xbool GetDefineFlagX (std::string_view name) const  noexcept;
     /// Returns string list flag, empty array if not exist
     const Array<std::string> & GetStringListFlag (const std::string & name) const;
     /// Returns num list flag, empty array if not exist
     const Array<double> & GetNumListFlag (const std::string & name) const;
     /// Returns flag list flag, empty flag if not exist
     const Flags & GetFlagsFlag (const std::string & name) const;
+    const std::any& GetAnyFlag (const std::string& name) const;
 
 
     /// Test, if string flag is defined
-    bool StringFlagDefined (const std::string & name) const;
+    bool StringFlagDefined (std::string_view name) const noexcept;
     /// Test, if num flag is defined
-    bool NumFlagDefined (const std::string & name) const;
+    bool NumFlagDefined (std::string_view name) const noexcept;
     /// Test, if num flag is defined
-    bool FlagsFlagDefined (const std::string & name) const;
+    bool FlagsFlagDefined (std::string_view name) const noexcept;
     /// Test, if string list flag is defined
-    bool StringListFlagDefined (const std::string & name) const;
+    bool StringListFlagDefined (std::string_view name) const noexcept;
     /// Test, if num list flag is defined
-    bool NumListFlagDefined (const std::string & name) const;
+    bool NumListFlagDefined (std::string_view name) const noexcept;
+    bool AnyFlagDefined (std::string_view name) const noexcept;
 
     /// number of string flags
     int GetNStringFlags () const { return strflags.Size(); }
@@ -160,6 +167,7 @@ namespace ngcore
     int GetNStringListFlags () const { return strlistflags.Size(); }
     /// number of num-list flags
     int GetNNumListFlags () const { return numlistflags.Size(); }
+    int GetNAnyFlags() const { return anyflags.Size(); }
 
     ///
     const std::string & GetStringFlag (int i, std::string & name) const
@@ -174,6 +182,8 @@ namespace ngcore
     { name = strlistflags.GetName(i); return strlistflags[i]; }
     const Flags & GetFlagsFlag (int i, std::string & name) const
     { name = flaglistflags.GetName(i); return flaglistflags[i]; }
+    const std::any& GetAnyFlag(int i, std::string& name) const
+    { name = anyflags.GetName(i); return anyflags[i]; }
   };
 
   /// Print flags

libsrc/core/generate_mpi_sources.py

@@ -0,0 +1,174 @@
+functions = [
+        ("double", "MPI_Wtime"),
+        ("int", "MPI_Allgather", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "MPI_Comm"),
+        ("int", "MPI_Allreduce", "void*", "void*", "int", "MPI_Datatype", "MPI_Op", "MPI_Comm"),
+        ("int", "MPI_Alltoall", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "MPI_Comm"),
+        ("int", "MPI_Barrier", "MPI_Comm"),
+        ("int", "MPI_Bcast", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Ibcast", "void*", "int", "MPI_Datatype", "int", "MPI_Comm", "MPI_Request*"),    
+        ("int", "MPI_Comm_c2f", "MPI_Comm"),
+        ("int", "MPI_Comm_create", "MPI_Comm", "MPI_Group", "MPI_Comm*"),
+        ("int", "MPI_Comm_create_group", "MPI_Comm", "MPI_Group", "int", "MPI_Comm*"),
+        ("int", "MPI_Comm_free", "MPI_Comm*"),
+        ("int", "MPI_Comm_group", "MPI_Comm", "MPI_Group*"),
+        ("int", "MPI_Comm_rank", "MPI_Comm", "int*"),
+        ("int", "MPI_Comm_size", "MPI_Comm", "int*"),
+        ("int", "MPI_Finalize"),
+        ("int", "MPI_Gather", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Gatherv", "void*", "int", "MPI_Datatype", "void*", "int*", "int*", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Get_count", "MPI_Status*", "MPI_Datatype", "int*"),
+        ("int", "MPI_Get_processor_name", "char*", "int*"),
+        ("int", "MPI_Group_incl", "MPI_Group", "int", "int*", "MPI_Group*"),
+        ("int", "MPI_Init", "int*", "char***"),
+        ("int", "MPI_Init_thread", "int*", "char***", "int", "int*"),
+        ("int", "MPI_Initialized", "int*"),
+        ("int", "MPI_Iprobe", "int", "int", "MPI_Comm", "int*", "MPI_Status*"),
+        ("int", "MPI_Irecv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Isend", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Probe", "int", "int", "MPI_Comm", "MPI_Status*"),
+        ("int", "MPI_Query_thread", "int*"),
+        ("int", "MPI_Recv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Status*"),
+        ("int", "MPI_Recv_init", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Reduce", "void*", "void*", "int", "MPI_Datatype", "MPI_Op", "int", "MPI_Comm"),
+        ("int", "MPI_Reduce_local", "void*", "void*", "int", "MPI_Datatype", "MPI_Op"),
+        ("int", "MPI_Request_free", "MPI_Request*"),
+        ("int", "MPI_Scatter", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"),
+        ("int", "MPI_Send", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm"),
+        ("int", "MPI_Send_init", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"),
+        ("int", "MPI_Startall", "int", "MPI_Request*:0"),
+        ("int", "MPI_Type_commit", "MPI_Datatype*"),
+        ("int", "MPI_Type_contiguous", "int", "MPI_Datatype", "MPI_Datatype*"),
+        ("int", "MPI_Type_create_resized", "MPI_Datatype", "MPI_Aint", "MPI_Aint", "MPI_Datatype*"),
+        ("int", "MPI_Type_create_struct", "int", "int*:0", "MPI_Aint*:0", "MPI_Datatype*:0", "MPI_Datatype*"),
+        ("int", "MPI_Type_free", "MPI_Datatype*"),
+        ("int", "MPI_Type_get_extent", "MPI_Datatype", "MPI_Aint*", "MPI_Aint*"),
+        ("int", "MPI_Type_indexed", "int", "int*:0", "int*:0", "MPI_Datatype", "MPI_Datatype*"),
+        ("int", "MPI_Type_size", "MPI_Datatype", "int*"),
+        ("int", "MPI_Wait", "MPI_Request*", "MPI_Status*"),
+        ("int", "MPI_Waitall", "int", "MPI_Request*:0", "MPI_Status*"),
+        ("int", "MPI_Waitany", "int", "MPI_Request*:0", "int*", "MPI_Status*"),
+        ]
+
+constants = [
+        ("MPI_Comm", "MPI_COMM_NULL"),
+        ("MPI_Comm", "MPI_COMM_WORLD"),
+        ("MPI_Datatype", "MPI_CHAR"),
+        ("MPI_Datatype", "MPI_CXX_DOUBLE_COMPLEX"),
+        ("MPI_Datatype", "MPI_C_BOOL"),
+        ("MPI_Datatype", "MPI_DATATYPE_NULL"),
+        ("MPI_Datatype", "MPI_DOUBLE"),
+        ("MPI_Datatype", "MPI_FLOAT"),
+        ("MPI_Datatype", "MPI_INT"),
+        ("MPI_Datatype", "MPI_SHORT"),
+        ("MPI_Datatype", "MPI_UINT64_T"),
+        ("MPI_Op", "MPI_LOR"),
+        ("MPI_Op", "MPI_MAX"),
+        ("MPI_Op", "MPI_MIN"),
+        ("MPI_Op", "MPI_SUM"),
+        ("MPI_Request", "MPI_REQUEST_NULL"),
+        ("MPI_Status*", "MPI_STATUSES_IGNORE"),
+        ("MPI_Status*", "MPI_STATUS_IGNORE"),
+        ("int", "MPI_ANY_SOURCE"),
+        ("int", "MPI_ANY_TAG"),
+        ("int", "MPI_MAX_PROCESSOR_NAME"),
+        ("int", "MPI_PROC_NULL"),
+        ("int", "MPI_ROOT"),
+        ("int", "MPI_SUBVERSION"),
+        ("int", "MPI_THREAD_MULTIPLE"),
+        ("int", "MPI_THREAD_SINGLE"),
+        ("int", "MPI_VERSION"),
+        ("void*", "MPI_IN_PLACE"),
+]
+
+def get_args(f, counts=False):
+    args = []
+    for arg in f[2:]:
+        has_count = ':' in arg
+        if has_count:
+            s, count = arg.split(':')
+            count = int(count)
+        else:
+            s = arg
+            count = None
+        if s.startswith("MPI_"):
+            s = "NG_" + s
+        if counts:
+            args.append((s, count))
+        else:
+            args.append(s)
+    return args
+
+def generate_declarations():
+    code = ""
+    nowrapper_code = ""
+    for f in functions:
+        ret = f[0]
+        name = f[1]
+        args = ", ".join(get_args(f))
+        code += f"NGCORE_API extern {ret} (*NG_{name})({args});\n"
+        nowrapper_code += f"#define NG_{name} {name}\n"
+
+    for typ, name in constants:
+        if typ.startswith("MPI_"):
+            typ = "NG_" + typ
+        code += f"NGCORE_API extern {typ} NG_{name};\n"
+        nowrapper_code += f"#define NG_{name} {name}\n"
+
+    with open("ng_mpi_generated_declarations.hpp", "w") as f:
+        f.write("#ifdef NG_MPI_WRAPPER\n")
+        f.write(code)
+        f.write("#else  // NG_MPI_WRAPPER\n")
+        f.write(nowrapper_code)
+        f.write("#endif // NG_MPI_WRAPPER\n")
+
+def generate_dummy_init():
+    code = ""
+    for f in functions:
+        ret = f[0]
+        name = f[1]
+        args = ", ".join(get_args(f))
+        code += f"decltype(NG_{name}) NG_{name} = []({args})->{ret} {{ throw no_mpi(); }};\n"
+
+    for typ, name in constants:
+        if typ.startswith("MPI_"):
+            typ = "NG_" + typ
+        code += f"{typ} NG_{name} = 0;\n"
+
+    with open("ng_mpi_generated_dummy_init.hpp", "w") as f:
+        f.write(code)
+
+def generate_init():
+    code = ""
+    for f in functions:
+        ret = f[0]
+        name = f[1]
+        args = get_args(f, counts=True)
+        in_args  =''
+        call_args = ''
+        for i in range(len(args)):
+            arg, count = args[i]
+            if i > 0:
+                in_args += ', '
+                call_args += ', '
+            in_args += arg + f" arg{i}"
+            if not arg.startswith("NG_"):
+                # plain type (like int, int *, etc.), just pass the argument along
+                call_args += f" arg{i}"
+            elif count is None:
+                # MPI type (by value or pointer), but just one object, no arrays
+                call_args += f" ng2mpi(arg{i})"
+            else:
+                # arrays of MPI types, we need to copy them due to incompatible size
+                call_args += f" ng2mpi(arg{i}, arg{count})"
+        code += f"NG_{name} = []({in_args})->{ret} {{ return {name}({call_args}); }};\n"
+
+    for _, name in constants:
+        code += f"NG_{name} = mpi2ng({name});\n"
+
+    with open("ng_mpi_generated_init.hpp", "w") as f:
+        f.write(code)
+
+if __name__ == "__main__":
+    generate_declarations()
+    generate_dummy_init()
+    generate_init()

libsrc/core/hashtable.hpp

@@ -9,8 +9,9 @@
 
 #include <string>
 #include <tuple>
+#include <optional>
 
-#include "mpi_wrapper.hpp"
+// #include "mpi_wrapper.hpp"
 #include "ngcore_api.hpp"
 #include "table.hpp"
 #include "utils.hpp"
@@ -37,52 +38,68 @@ namespace ngcore
   };
   
   
-  
 
+  // feature check macro for transition from INT to IVec
+#define NGCORE_HAS_IVEC
+  
   /// N integers
   template <int N, typename T = int>
-  class INT
+  class IVec
   {
     /// data
-    T i[(N>0)?N:1];
+    // T i[(N>0)?N:1];
 
+    HTArray<N,T> i;
+    
   public:
     ///
-    NETGEN_INLINE INT () { }
+    constexpr NETGEN_INLINE IVec () = default;
+    constexpr NETGEN_INLINE IVec (const IVec & i1) : i(i1.i) { }
 
+    constexpr NETGEN_INLINE IVec (T ai1) : i(ai1) { }
+    
+    template <class... T2,
+              std::enable_if_t<N==1+sizeof...(T2),bool> = true>
+    constexpr IVec (const T &v, T2... rest)
+      : i{v,rest...} { } 
+
+    /*
     /// init all
-    NETGEN_INLINE INT (T ai1)
+    NETGEN_INLINE IVec (T ai1)
     { 
-      for (int j = 0; j < N; j++) { i[j] = ai1; }
+     for (int j = 0; j < N; j++) { i[j] = ai1; }
     }
 
     /// init i[0], i[1]
-    NETGEN_INLINE INT (T ai1, T ai2)
-    { i[0] = ai1; i[1] = ai2; }
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2)
+      : i{ai1, ai2} { ; } 
 
     /// init i[0], i[1], i[2]
-    NETGEN_INLINE INT (T ai1, T ai2, T ai3)
-    { i[0] = ai1; i[1] = ai2; i[2] = ai3; }
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2, T ai3)
+      : i{ai1, ai2, ai3} { ; } 
 
     /// init i[0], i[1], i[2]
-    NETGEN_INLINE INT (T ai1, T ai2, T ai3, T ai4)
-    { i[0] = ai1; i[1] = ai2; i[2] = ai3; i[3] = ai4; }
-
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2, T ai3, T ai4)
+      : i{ai1, ai2, ai3, ai4} { ; }
+    
     /// init i[0], i[1], i[2]
-    NETGEN_INLINE INT (T ai1, T ai2, T ai3, T ai4, T ai5)
-    { i[0] = ai1; i[1] = ai2; i[2] = ai3; i[3] = ai4; i[4] = ai5;}
-
+    constexpr NETGEN_INLINE IVec (T ai1, T ai2, T ai3, T ai4, T ai5)
+      : i{ai1, ai2, ai3, ai4, ai5} { ; }      
+      
     /// init i[0], i[1], i[2]
-    NETGEN_INLINE INT (T ai1, T ai2, T ai3, T ai4, T ai5, T ai6, T ai7, T ai8, T ai9)
-    { i[0] = ai1; i[1] = ai2; i[2] = ai3; i[3] = ai4; i[4] = ai5; i[5] = ai6; i[6] = ai7; i[7] = ai8; i[8] = ai9; }
-
-    void DoArchive(Archive& ar)
+    NETGEN_INLINE IVec (T ai1, T ai2, T ai3, T ai4, T ai5, T ai6, T ai7, T ai8, T ai9)
+      : i{ai1, ai2, ai3, ai4, ai5, ai6, ai7, ai8, ai9 } { ; }            
+    */
+    
+    template <typename ARCHIVE>
+    void DoArchive(ARCHIVE& ar)
     {
-      ar.Do(i, N);
+      // ar.Do(i.begin(), N);
+      ar.Do(i.Ptr(), N);
     }
 
     template <int N2, typename T2>
-    NETGEN_INLINE INT (const INT<N2,T2> & in2)
+    NETGEN_INLINE IVec (const IVec<N2,T2> & in2)
     {
       if (N2 <= N)
         {
@@ -99,7 +116,7 @@ namespace ngcore
     }
 
     template <typename T2>
-    NETGEN_INLINE INT (const BaseArrayObject<T2> & ao)
+    NETGEN_INLINE IVec (const BaseArrayObject<T2> & ao)
     {
       for (int j = 0; j < N; j++)
         i[j] = ao.Spec()[j];
@@ -107,7 +124,7 @@ namespace ngcore
     
     NETGEN_INLINE size_t Size() const { return N; }
     /// all ints equal ?
-    NETGEN_INLINE bool operator== (const INT & in2) const
+    NETGEN_INLINE bool operator== (const IVec & in2) const
     { 
       for (int j = 0; j < N; j++) 
 	if (i[j] != in2.i[j]) return 0;
@@ -115,7 +132,7 @@ namespace ngcore
     }
 
     /// any ints unequal ?
-    NETGEN_INLINE bool operator!= (const INT & in2) const
+    NETGEN_INLINE bool operator!= (const IVec & in2) const
     {
       for (int j = 0; j < N; j++)
         if (i[j] != in2.i[j]) return 1;
@@ -123,7 +140,7 @@ namespace ngcore
     }
 
     /// sort integers
-    NETGEN_INLINE INT & Sort () & 
+    NETGEN_INLINE IVec & Sort () & 
     {
       for (int k = 0; k < N; k++)
 	for (int l = k+1; l < N; l++)
@@ -132,7 +149,7 @@ namespace ngcore
       return *this;
     }
 
-    NETGEN_INLINE INT Sort () &&
+    NETGEN_INLINE IVec Sort () &&
     {
       for (int k = 0; k < N; k++)
 	for (int l = k+1; l < N; l++)
@@ -146,15 +163,15 @@ namespace ngcore
     { return i[j]; }
 
     /// access
-    NETGEN_INLINE const T & operator[] (int j) const
+    NETGEN_INLINE constexpr const T & operator[] (int j) const
     { return i[j]; }
 
     template <size_t J>
-    T get() const { return i[J]; }
+    constexpr T get() const { return i[J]; }
     
     operator FlatArray<T> () { return FlatArray<T> (N, &i[0]); } 
 
-    NETGEN_INLINE INT<N,T> & operator= (T value)
+    NETGEN_INLINE IVec<N,T> & operator= (T value)
     {
       for (int j = 0; j < N; j++)
 	i[j] = value;
@@ -162,7 +179,7 @@ namespace ngcore
     }
 
     template <typename T2>
-    NETGEN_INLINE INT<N,T> & operator= (INT<N,T2> v2)
+    NETGEN_INLINE IVec<N,T> & operator= (IVec<N,T2> v2)
     {
       for (int j = 0; j < N; j++)
 	i[j] = v2[j];
@@ -174,18 +191,25 @@ namespace ngcore
     {
       return MakeTupleFromInt<N>()(*this);
     }
+
+    bool Contains (T val)
+    {
+      for (int j = 0; j < N; j++)
+        if (i[j] == val) return true;
+      return false;
+    }
   };
 
   /// sort 2 integers
   template <>
-  NETGEN_INLINE INT<2> & INT<2>::Sort () & 
+  NETGEN_INLINE IVec<2> & IVec<2>::Sort () & 
   {
     if (i[0] > i[1]) Swap (i[0], i[1]);
     return *this;
   }
 
   template <>
-  NETGEN_INLINE INT<2> INT<2>::Sort () &&
+  NETGEN_INLINE IVec<2> IVec<2>::Sort () &&
   {
     if (i[0] > i[1]) Swap (i[0], i[1]);
     return *this;
@@ -193,7 +217,7 @@ namespace ngcore
 
   /// sort 3 integers
   template <>
-  NETGEN_INLINE INT<3> INT<3>::Sort () &&
+  NETGEN_INLINE IVec<3> IVec<3>::Sort () &&
   {
     if (i[0] > i[1]) Swap (i[0], i[1]);
     if (i[1] > i[2]) Swap (i[1], i[2]);
@@ -203,7 +227,7 @@ namespace ngcore
 
   /// Print integers
   template <int N, typename T>
-  inline ostream & operator<<(ostream  & s, const INT<N,T> & i2)
+  inline ostream & operator<<(ostream  & s, const IVec<N,T> & i2)
   {
     for (int j = 0; j < N; j++)
       s << (int) i2[j] << " ";
@@ -211,15 +235,15 @@ namespace ngcore
   }
   
   template <int N, typename T>
-  auto begin(const INT<N,T> & ind)
+  auto begin(const IVec<N,T> & ind)
   {
-    return AOWrapperIterator<INT<N,T>> (ind, 0);
+    return AOWrapperIterator<IVec<N,T>> (ind, 0);
   }
 
   template <int N, typename T>
-  auto end(const INT<N,T> & ind)
+  auto end(const IVec<N,T> & ind)
   {
-    return AOWrapperIterator<INT<N,T>> (ind, N);    
+    return AOWrapperIterator<IVec<N,T>> (ind, N);    
   }
 
 
@@ -228,9 +252,9 @@ namespace ngcore
 
   
   template <int N, typename TI>
-  NETGEN_INLINE size_t HashValue (const INT<N,TI> & ind, size_t size)
+  NETGEN_INLINE size_t HashValue (const IVec<N,TI> & ind, size_t size)
   {
-    INT<N,size_t> lind = ind;    
+    IVec<N,size_t> lind = ind;    
     size_t sum = 0;
     for (int i = 0; i < N; i++)
       sum += lind[i];
@@ -239,24 +263,24 @@ namespace ngcore
 
   /// hash value of 1 int
   template <typename TI>
-  NETGEN_INLINE size_t HashValue (const INT<1,TI> & ind, size_t size) 
+  NETGEN_INLINE size_t HashValue (const IVec<1,TI> & ind, size_t size) 
   {
     return ind[0] % size;
   }
 
   /// hash value of 2 int
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue (const INT<2,TI> & ind, size_t size) 
+  NETGEN_INLINE size_t HashValue (const IVec<2,TI> & ind, size_t size) 
   {
-    INT<2,size_t> lind = ind;
+    IVec<2,size_t> lind = ind;
     return (113*lind[0]+lind[1]) % size;
   }
 
   /// hash value of 3 int
   template <typename TI>    
-  NETGEN_INLINE size_t HashValue (const INT<3,TI> & ind, size_t size) 
+  NETGEN_INLINE size_t HashValue (const IVec<3,TI> & ind, size_t size) 
   {
-    INT<3,size_t> lind = ind;
+    IVec<3,size_t> lind = ind;
     return (113*lind[0]+59*lind[1]+lind[2]) % size;
   }
 
@@ -276,9 +300,9 @@ namespace ngcore
 
   
   template <int N, typename TI>
-  NETGEN_INLINE size_t HashValue2 (const INT<N,TI> & ind, size_t mask)
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<N,TI> & ind, size_t mask)
   {
-    INT<N,size_t> lind = ind;    
+    IVec<N,size_t> lind = ind;    
     size_t sum = 0;
     for (int i = 0; i < N; i++)
       sum += lind[i];
@@ -287,32 +311,32 @@ namespace ngcore
 
   /// hash value of 1 int
   template <typename TI>
-  NETGEN_INLINE size_t HashValue2 (const INT<1,TI> & ind, size_t mask) 
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<1,TI> & ind, size_t mask) 
   {
     return ind[0] & mask;
   }
 
   /// hash value of 2 int
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue2 (const INT<2,TI> & ind, size_t mask) 
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<2,TI> & ind, size_t mask) 
   {
-    INT<2,size_t> lind = ind;
+    IVec<2,size_t> lind = ind;
     return (113*lind[0]+lind[1]) & mask;
   }
 
   /// hash value of 3 int
   template <typename TI>    
-  NETGEN_INLINE size_t HashValue2 (const INT<3,TI> & ind, size_t mask) 
+  NETGEN_INLINE constexpr size_t HashValue2 (const IVec<3,TI> & ind, size_t mask) 
   {
-    INT<3,size_t> lind = ind;
+    IVec<3,size_t> lind = ind;
     return (113*lind[0]+59*lind[1]+lind[2]) & mask;
   }
 
-  NETGEN_INLINE size_t HashValue2 (size_t ind, size_t mask)
+  NETGEN_INLINE constexpr size_t HashValue2 (size_t ind, size_t mask)
   {
     return ind & mask;
   }
-  NETGEN_INLINE size_t HashValue2 (int ind, size_t mask)
+  NETGEN_INLINE constexpr size_t HashValue2 (int ind, size_t mask)
   {
     return size_t(ind) & mask;
   }
@@ -324,7 +348,7 @@ namespace ngcore
   // using ngstd::max;
 
   template <int D, typename T>
-  NETGEN_INLINE T Max (const INT<D,T> & i)
+  NETGEN_INLINE T Max (const IVec<D,T> & i)
   {
     if (D == 0) return 0;
     T m = i[0];
@@ -334,7 +358,7 @@ namespace ngcore
   }
 
   template <int D, typename T>
-  NETGEN_INLINE T Min (const INT<D,T> & i)
+  NETGEN_INLINE T Min (const IVec<D,T> & i)
   {
     if (D == 0) return 0;
     T m = i[0];
@@ -344,18 +368,18 @@ namespace ngcore
   }
 
   template <int D, typename T>
-  NETGEN_INLINE INT<D,T> Max (INT<D,T> i1, INT<D,T> i2)
+  NETGEN_INLINE IVec<D,T> Max (IVec<D,T> i1, IVec<D,T> i2)
   {
-    INT<D,T> tmp;
+    IVec<D,T> tmp;
     for (int i = 0; i < D; i++)
       tmp[i] = std::max(i1[i], i2[i]);
     return tmp;
   }
 
   template <int D, typename T>
-  NETGEN_INLINE INT<D,T> operator+ (INT<D,T> i1, INT<D,T> i2)
+  NETGEN_INLINE IVec<D,T> operator+ (IVec<D,T> i1, IVec<D,T> i2)
   {
-    INT<D,T> tmp;
+    IVec<D,T> tmp;
     for (int i = 0; i < D; i++)
       tmp[i] = i1[i]+i2[i];
     return tmp;
@@ -493,7 +517,7 @@ namespace ngcore
       for (int i = 0; i < table[bnr].Size(); i++)
 	if (table[bnr][i].first == ind)
 	  return i;
-      throw Exception ("Ask for unsused hash-value");
+      throw Exception ("Ask for unused hash-value");
     }
 
     T & operator[] (T_HASH ahash)
@@ -567,7 +591,27 @@ namespace ngcore
     return res; 
   }
 
+  template <typename T>
+  constexpr inline T InvalidHash() { return T(-1); }
 
+  template <typename T_HASH>
+  struct CHT_trait
+  {
+    constexpr static inline T_HASH Invalid() { return InvalidHash<T_HASH>(); }
+    constexpr static inline size_t HashValue (const T_HASH & hash, size_t mask) { return HashValue2(hash, mask); }
+  };
+
+  template <typename T1, typename T2>
+  struct CHT_trait<std::tuple<T1,T2>>
+  {
+    constexpr static inline std::tuple<T1,T2> Invalid() { return { CHT_trait<T1>::Invalid(), CHT_trait<T2>::Invalid() } ; }
+    constexpr static inline size_t HashValue (const std::tuple<T1,T2> & hash, size_t mask)
+    {
+      return (CHT_trait<T1>::HashValue(std::get<0>(hash), mask) + CHT_trait<T2>::HashValue(std::get<1>(hash),mask)) & mask;
+    }
+  };
+
+  
 
   /**
      A closed hash-table.
@@ -588,14 +632,18 @@ namespace ngcore
     ///
     Array<T> cont;
     ///
-    T_HASH invalid = -1;
+    // T_HASH invalid = -1;
+    // static constexpr T_HASH invalid = InvalidHash<T_HASH>();
+    static constexpr T_HASH invalid = CHT_trait<T_HASH>::Invalid();
   public:
     ///
     ClosedHashTable (size_t asize = 128)
       : size(RoundUp2(asize)), hash(size), cont(size)
     {
       mask = size-1;
-      hash = T_HASH(invalid);
+      // hash = T_HASH(invalid);
+      // hash = InvalidHash<T_HASH>();
+      hash = CHT_trait<T_HASH>::Invalid();
     }
 
     ClosedHashTable (ClosedHashTable && ht2) = default;
@@ -604,7 +652,8 @@ namespace ngcore
     ClosedHashTable (size_t asize, LocalHeap & lh)
       : size(RoundUp2(asize)), mask(size-1), hash(size, lh), cont(size, lh)
     {
-      hash = T_HASH(invalid);
+      // hash = T_HASH(invalid);
+      hash = InvalidHash<T_HASH>();
     }
 
     ClosedHashTable & operator= (ClosedHashTable && ht2) = default;
@@ -625,24 +674,17 @@ namespace ngcore
     size_t UsedElements () const
     {
       return used;
-      /*
-      size_t cnt = 0;
-      for (size_t i = 0; i < size; i++)
-	if (hash[i] != invalid)
-	  cnt++;
-      return cnt;
-      */
     }
 
     size_t Position (const T_HASH ind) const
     {
-      size_t i = HashValue2(ind, mask);
-      while (1)
+      // size_t i = HashValue2(ind, mask);
+      size_t i = CHT_trait<T_HASH>::HashValue(ind, mask);
+      while (true)
 	{
 	  if (hash[i] == ind) return i;
 	  if (hash[i] == invalid) return size_t(-1);
-	  i++;
-	  if (i >= size) i = 0;
+          i = (i+1) & mask;          
 	}
     }
 
@@ -659,9 +701,10 @@ namespace ngcore
     {
       if (UsedElements()*2 > Size()) DoubleSize();
       
-      size_t i = HashValue2 (ind, mask);
+      // size_t i = HashValue2 (ind, mask);
+      size_t i = CHT_trait<T_HASH>::HashValue (ind, mask);
 
-      while (1)
+      while (true)
 	{
 	  if (hash[i] == invalid)
 	    { 
@@ -675,8 +718,7 @@ namespace ngcore
 	      apos = i; 
 	      return false; 
 	    }
-	  i++;
-	  if (i >= size) i = 0;
+          i = (i+1) & mask;
 	}
     }
 
@@ -705,6 +747,16 @@ namespace ngcore
       return (Position (ahash) != size_t(-1));
     }
 
+    inline std::optional<T> GetIfUsed (const T_HASH & ahash) const
+    {
+      size_t pos = Position (ahash);
+      if (pos != size_t(-1))
+        return cont[pos];
+      else
+        return std::nullopt;
+    }
+    
+
     void SetData (size_t pos, const T_HASH & ahash, const T & acont)
     {
       hash[pos] = ahash;
@@ -727,6 +779,11 @@ namespace ngcore
       acont = cont[pos];
     }
 
+    T GetData (size_t pos) const
+    {
+      return cont[pos];
+    }
+
     std::pair<T_HASH,T> GetBoth (size_t pos) const
     {
       return std::pair<T_HASH,T> (hash[pos], cont[pos]);
@@ -771,6 +828,21 @@ namespace ngcore
           pos = nextpos;
         }
     }
+
+    void DeleteData()
+    {
+      hash = T_HASH(invalid);
+      used = 0;
+    }
+
+    template <typename ARCHIVE>
+    void DoArchive (ARCHIVE& ar)
+    {
+      ar & hash & cont;
+      ar & size & mask & used;
+    }    
+
+    struct EndIterator { };
     
     class Iterator
     {
@@ -788,24 +860,21 @@ namespace ngcore
         while (nr < tab.Size() && !tab.UsedPos(nr)) nr++;
         return *this;
       }
-      bool operator!= (const Iterator & it2) { return nr != it2.nr; }
-      auto operator* () const
-      {
-        T_HASH hash;
-        T val;
-        tab.GetData(nr, hash,val);
-        return std::make_pair(hash,val);
-      }
+
+      bool operator!= (EndIterator it2) { return nr != tab.Size(); }
+      
+      auto operator* () const { return tab.GetBoth(nr); }
     };
 
     Iterator begin() const { return Iterator(*this, 0); }
-    Iterator end() const { return Iterator(*this, Size()); } 
+    EndIterator end() const { return EndIterator(); }
   };
 
   template <class T_HASH, class T>  
   ostream & operator<< (ostream & ost,
                         const ClosedHashTable<T_HASH,T> & tab)
   {
+    /*
     for (size_t i = 0; i < tab.Size(); i++)
       if (tab.UsedPos(i))
         {
@@ -814,25 +883,28 @@ namespace ngcore
           tab.GetData (i, key, val);
           ost << key << ": " << val << ", ";
         }
+    */
+    for (auto [key,val] : tab)
+      ost << key << ": " << val << ", ";      
     return ost;
   }
 
   template <typename TI>
-  NETGEN_INLINE size_t HashValue (const INT<3,TI> ind)
+  NETGEN_INLINE size_t HashValue (const IVec<3,TI> ind)
   {
-    INT<3,size_t> lind = ind;
+    IVec<3,size_t> lind = ind;
     return 113*lind[0]+59*lind[1]+lind[2];
   }
 
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue (const INT<2,TI> ind)
+  NETGEN_INLINE size_t HashValue (const IVec<2,TI> ind)
   {
-    INT<2,size_t> lind = ind;
+    IVec<2,size_t> lind = ind;
     return 113*lind[0]+lind[1];
   }
 
   template <typename TI>  
-  NETGEN_INLINE size_t HashValue (const INT<1,TI> ind)
+  NETGEN_INLINE size_t HashValue (const IVec<1,TI> ind)
   {
     return ind[0];
   }
@@ -1059,22 +1131,113 @@ namespace ngcore
   }
 
 
-    
-  template <int N, typename T>
-  Archive & operator & (Archive & archive, INT<N,T> & mi)
+
+
+
+
+
+
+
+  template <class T, class IndexType>
+  class CompressedTable
   {
-    for (int i = 0; i < N; i++)
-      archive & mi[i];
-    return archive;
-  }
+    Table<T, size_t> table;
+    ClosedHashTable<IndexType, size_t> idmap;
+    
+  public:
+    CompressedTable (Table<T, size_t> && atable, ClosedHashTable<IndexType, size_t> && aidmap)
+      : table(std::move(atable)), idmap(std::move(aidmap)) { }
+
+    FlatArray<T> operator[](IndexType id) const
+    {
+      if (auto nr = idmap.GetIfUsed(id))
+        return table[*nr];
+      else
+        return { 0, nullptr };
+    }
+    auto & GetTable() { return table; }
+  };
+
+
+  template <class T, typename IndexType>
+  class CompressedTableCreator
+  {
+  protected:
+    int mode;    // 1 .. cnt, 2 .. cnt entries, 3 .. fill table
+    size_t nd;   // number of entries;
+    ClosedHashTable<IndexType, size_t> idmap;
+    Array<int,size_t> cnt;
+    Table<T,size_t> table;
+  public:
+    CompressedTableCreator()
+    { nd = 0; mode = 1; }
+
+    CompressedTable<T,IndexType> MoveTable()
+    {
+      return { std::move(table), std::move(idmap) };
+    }
+
+    bool Done () { return mode > 3; }
+    void operator++(int) { SetMode (mode+1); }
+
+    int GetMode () const { return mode; }
+    void SetMode (int amode)
+    {
+      mode = amode;
+      if (mode == 2)
+	{
+          cnt.SetSize(nd);  
+          cnt = 0;
+	}
+      if (mode == 3)
+	{
+          table = Table<T,size_t> (cnt);
+          cnt = 0;
+	}
+    }
+
+    void Add (IndexType blocknr, const T & data)
+    {
+      switch (mode)
+	{
+	case 1:
+          {
+            if (!idmap.Used (blocknr))
+              idmap[blocknr] = nd++;
+            break;
+          }
+	case 2:
+	  cnt[idmap.Get(blocknr)]++;
+	  break;
+	case 3:
+          size_t cblock = idmap.Get(blocknr);
+          int ci = cnt[cblock]++;
+          table[cblock][ci] = data;
+	  break;
+	}
+    }
+  };
+
+
+  
+
+
+
+
+
+
+
+
+
+  
 } // namespace ngcore
 
 
-
+/*
 #ifdef PARALLEL
 namespace ngcore {
   template<int S, typename T>
-  class MPI_typetrait<ngcore::INT<S, T> >
+  class MPI_typetrait<ngcore::IVec<S, T> >
   {
   public:
     /// gets the MPI datatype
@@ -1091,6 +1254,19 @@ namespace ngcore {
   };
 }
 #endif
+*/
+
+namespace ngcore
+{
+  template<typename T> struct MPI_typetrait;
+  
+  template<int S, typename T>
+  struct MPI_typetrait<IVec<S, T> > {
+    static auto MPIType () {
+      return MPI_typetrait<std::array<T,S>>::MPIType();
+    }
+  };
+}
 
 
 
@@ -1098,8 +1274,8 @@ namespace std
 {
   // structured binding support
   template <auto N, typename T>
-  struct tuple_size<ngcore::INT<N,T>> : std::integral_constant<std::size_t, N> {};
-  template<size_t N, auto M, typename T> struct tuple_element<N,ngcore::INT<M,T>> { using type = T; };
+  struct tuple_size<ngcore::IVec<N,T>> : std::integral_constant<std::size_t, N> {};
+  template<size_t N, auto M, typename T> struct tuple_element<N,ngcore::IVec<M,T>> { using type = T; };
 }
 
 #endif

libsrc/core/localheap.hpp

@@ -136,7 +136,7 @@ public:
 
 
   /// free memory
-  NETGEN_INLINE virtual ~LocalHeap ()
+  virtual ~LocalHeap ()
   {
     if (owner)
       delete [] data;
@@ -163,7 +163,7 @@ public:
   }
 
   /// allocates size bytes of memory from local heap
-  NETGEN_INLINE void * Alloc (size_t size) final // throw (LocalHeapOverflow)
+  void * Alloc (size_t size) final // throw (LocalHeapOverflow)
   {
     char * oldp = p;
   
@@ -181,7 +181,7 @@ public:
 
     /// allocates size objects of type T on local heap
     template <typename T>
-    NETGEN_INLINE T * Alloc (size_t size) // throw (LocalHeapOverflow)
+    T * Alloc (size_t size) // throw (LocalHeapOverflow)
     {
       char * oldp = p;
       size *= sizeof (T);
@@ -198,9 +198,9 @@ public:
       return reinterpret_cast<T*> (oldp);
     }
 
-    virtual void Delete(void* p) {}
+  virtual void Delete(void* /* p */) {}
 
-    virtual void ArrayDelete(void* p) {}
+  virtual void ArrayDelete(void* /* p */) {}
   private: 
     ///
 #ifndef __CUDA_ARCH__
@@ -211,7 +211,7 @@ public:
 
   public:
     /// free memory (dummy function)
-    NETGEN_INLINE void Free (void * data) throw () 
+  NETGEN_INLINE void Free (void * /* data */) throw () 
     {
       ;
     }

libsrc/core/logging.cpp

@@ -1,15 +1,6 @@
 #include "logging.hpp"
 
-#ifdef NETGEN_USE_SPDLOG
-
-#include <spdlog/spdlog.h>
-#include <spdlog/sinks/ansicolor_sink.h>
-#include <spdlog/sinks/basic_file_sink.h>
-
-#else // NETGEN_USE_SPDLOG
 #include <iostream>
-#endif // NETGEN_USE_SPDLOG
-
 
 namespace ngcore
 {
@@ -19,94 +10,10 @@ namespace ngcore
 
   void Logger::log(level::level_enum level, std::string && s)
   {
-#ifdef NETGEN_USE_SPDLOG
-    logger->log(spdlog::level::level_enum(level), s);
-#else // NETGEN_USE_SPDLOG
     if(level>=global_level)
       std::clog << s << '\n';
-#endif // NETGEN_USE_SPDLOG
   }
 
-#ifdef NETGEN_USE_SPDLOG
-  namespace detail
-  {
-    std::vector<std::shared_ptr<spdlog::sinks::sink>>& GetDefaultSinks()
-    {
-      static std::vector<std::shared_ptr<spdlog::sinks::sink>> sinks =
-        { std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>() };
-      return sinks;
-    }
-    std::shared_ptr<spdlog::logger> CreateDefaultLogger(const std::string& name)
-    {
-      auto& default_sinks = GetDefaultSinks();
-      auto logger = std::make_shared<spdlog::logger>(name, default_sinks.begin(), default_sinks.end());
-      spdlog::details::registry::instance().register_and_init(logger);
-      return logger;
-    }
-  } // namespace detail
-
-  std::shared_ptr<Logger> GetLogger(const std::string& name)
-  {
-    auto logger = spdlog::get(name);
-    if(!logger)
-      logger = detail::CreateDefaultLogger(name);
-    return std::make_shared<Logger>(logger);
-  }
-
-  void SetLoggingLevel(spdlog::level::level_enum level, const std::string& name)
-  {
-    if(!name.empty())
-      spdlog::get(name)->set_level(level);
-    else
-      spdlog::set_level(level);
-  }
-
-  void AddFileSink(const std::string& filename, spdlog::level::level_enum level, const std::string& logger)
-  {
-    auto sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(filename);
-    sink->set_level(level);
-    if(!logger.empty())
-      GetLogger(logger)->logger->sinks().push_back(sink);
-    else
-      {
-        detail::GetDefaultSinks().push_back(sink);
-        spdlog::details::registry::instance().apply_all([sink](auto logger) { logger->sinks().push_back(sink); });
-      }
-  }
-
-  void AddConsoleSink(spdlog::level::level_enum level, const std::string& logger)
-  {
-    auto sink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>();
-    sink->set_level(level);
-    if(!logger.empty())
-      GetLogger(logger)->logger->sinks().push_back(sink);
-    else
-      {
-        detail::GetDefaultSinks().push_back(sink);
-        spdlog::details::registry::instance().apply_all([sink](auto logger) { logger->sinks().push_back(sink); });
-      }
-  }
-
-  void ClearLoggingSinks(const std::string& logger)
-  {
-    if(!logger.empty())
-      GetLogger(logger)->logger->sinks().clear();
-    else
-      {
-        detail::GetDefaultSinks().clear();
-        spdlog::details::registry::instance().apply_all([](auto logger) { logger->sinks().clear(); });
-      }
-  }
-
-  void FlushOnLoggingLevel(spdlog::level::level_enum level, const std::string& logger)
-  {
-    if(!logger.empty())
-      GetLogger(logger)->logger->flush_on(level);
-    else
-      spdlog::flush_on(level);
-  }
-
-#else // NETGEN_USE_SPDLOG
 } //namespace ngcore
 
 namespace spdlog
@@ -140,5 +47,3 @@ namespace ngcore
   void ClearLoggingSinks(const std::string& /*unused*/) {}
   void FlushOnLoggingLevel(level::level_enum /*unused*/, const std::string& /*unused*/) {}
 } //namespace ngcore
-
-#endif // NETGEN_USE_SPDLOG

libsrc/core/logging.hpp

@@ -1,7 +1,6 @@
 #ifndef NETGEN_CORE_LOGGING_HPP
 #define NETGEN_CORE_LOGGING_HPP
 
-#undef NETGEN_USE_SPDLOG
 #include <iostream>
 #include <memory>
 #include <string>
@@ -11,15 +10,6 @@
 #include "ngcore_api.hpp"
 #include "utils.hpp"
 
-#ifdef NETGEN_USE_SPDLOG
-#include <spdlog/fmt/fmt.h>
-#include <spdlog/fmt/ostr.h> // to be able to parse anything to logger that implements operator << ostream
-#ifdef NETGEN_LOG_DEBUG
-#define SPDLOG_DEBUG_ON
-#define NETGEN_DEBUG_LOG(logger, ...) SPDLOG_DEBUG(logger, __VA_ARGS__)
-#endif // NETGEN_LOG_DEBUG
-#endif // NETGEN_USE_SPDLOG
-
 #ifndef NETGEN_DEBUG_LOG
 #define NETGEN_DEBUG_LOG(logger, ...)
 #endif  // NETGEN_DEBUG_LOG
@@ -52,7 +42,12 @@ namespace ngcore
     static NGCORE_API level::level_enum global_level;
 
   public:
-    static void SetGlobalLoggingLevel( level::level_enum level ) { global_level = level; }
+    static auto SetGlobalLoggingLevel( level::level_enum level )
+    {
+      auto oldval = global_level;
+      global_level = level;
+      return oldval;
+    }
 
     std::shared_ptr<spdlog::logger> logger;
 
@@ -60,13 +55,6 @@ namespace ngcore
 
     void NGCORE_API log( level::level_enum level, std::string && s);
 
-#ifdef NETGEN_USE_SPDLOG
-    template<typename ... Args>
-    void log( level::level_enum level, const char* str, Args ... args)
-    {
-      log(level, fmt::format(str, args...));
-    }
-#else // NETGEN_USE_SPDLOG
     template<typename T>
     std::string replace(std::string s, const T & t)
     {
@@ -100,7 +88,6 @@ namespace ngcore
     {
       log(level, log_helper(std::string(str), args...));
     }
-#endif // NETGEN_USE_SPDLOG
 
     template<typename ... Args>
     void trace( const char* str, Args ... args) { log(level::level_enum::trace, str, args...); }

libsrc/core/memtracer.hpp

@@ -0,0 +1,221 @@
+#ifndef NETGEN_CORE_MEMTRACER_HPP
+#define NETGEN_CORE_MEMTRACER_HPP
+
+#include <array>
+#include <chrono>
+#include <string>
+
+#include "array.hpp"
+#include "logging.hpp"
+#include "paje_trace.hpp"
+#include "utils.hpp"
+
+namespace ngcore
+{
+
+  class MemoryTracer;
+
+  namespace detail
+  {
+    //Type trait to check if a class implements a 'void SetMemoryTacing(int)' function
+    template<typename T>
+    struct has_StartMemoryTracing
+    {
+    private:
+      template<typename T2>
+      static constexpr auto check(T2*) ->
+        typename std::is_same<decltype(std::declval<T2>().StartMemoryTracing()),void>::type;
+      template<typename>
+      static constexpr std::false_type check(...);
+      using type = decltype(check<T>(nullptr)); // NOLINT
+    public:
+      static constexpr bool value = type::value;
+    };
+  } // namespace detail
+
+  class MemoryTracer
+  {
+#if defined(NETGEN_TRACE_MEMORY) && !defined(__CUDA_ARCH__)
+    NGCORE_API static std::vector<std::string> names;
+    NGCORE_API static std::vector<int> parents;
+
+    #if defined(NETGEN_CHECK_RANGE)
+    NGCORE_API static std::atomic<size_t> total_memory;
+    mutable size_t allocated_memory = 0;
+    #endif // NETGEN_CHECK_RANGE
+
+    static int CreateId(const std::string& name = "")
+    {
+      int id = names.size();
+      names.push_back(name);
+      parents.push_back(0);
+      if(id==10*8*1024)
+        std::cerr << "Allocated " << id << " MemoryTracer objects" << std::endl;
+      return id;
+    }
+    mutable int id = 0;
+
+    public:
+
+    MemoryTracer( std::string name )
+    {
+      id = CreateId(name);
+    }
+
+    MemoryTracer() { }
+
+    MemoryTracer(const MemoryTracer & tracer)
+    {
+      (*this) = tracer;
+    }
+
+    MemoryTracer(MemoryTracer && tracer)
+    {
+      (*this) = std::move(tracer);
+    }
+
+    MemoryTracer & operator=(const MemoryTracer & tracer) {
+      if(tracer.id)
+        id = CreateId(names[tracer.id]);
+      return *this;
+    }
+
+    MemoryTracer & operator=(MemoryTracer && tracer) {
+      ngcore::Swap(id, tracer.id);
+
+      #if defined(NETGEN_CHECK_RANGE)
+      ngcore::Swap(allocated_memory, tracer.allocated_memory);
+      #endif // NETGEN_CHECK_RANGE
+
+      return *this;
+    }
+
+    template <typename... TRest>
+    MemoryTracer( std::string name, TRest & ... rest )
+    {
+      id = CreateId(name);
+      Track(rest...);
+    }
+
+    #if defined(NETGEN_CHECK_RANGE)
+    // check if all memory was freed when object is destroyed
+    ~MemoryTracer()
+    {
+      NETGEN_CHECK_SAME(allocated_memory, 0);
+    }
+    #endif // NETGEN_CHECK_RANGE
+
+    NETGEN_INLINE void Alloc(size_t size) const
+    {
+      #if defined(NETGEN_CHECK_RANGE)
+      // Trace also nameless Memtracer objects if range checks are active
+      if(!id && size)
+        id = CreateId();
+      #endif // NETGEN_CHECK_RANGE
+
+      if(id && trace)
+        trace->AllocMemory(id, size);
+
+      #if defined(NETGEN_CHECK_RANGE)
+      if(id)
+      {
+        allocated_memory += size;
+        total_memory += size;
+      }
+      #endif // NETGEN_CHECK_RANGE
+    }
+
+    void Free(size_t size) const
+    {
+      if(id && trace)
+        trace->FreeMemory(id, size);
+
+      #if defined(NETGEN_CHECK_RANGE)
+      if(id)
+      {
+        // check if we have at least size bytes of memory currently allocated (such that allocated_memory doesn't get negative)
+        NETGEN_CHECK_RANGE(allocated_memory, static_cast<ptrdiff_t>(size), std::numeric_limits<ptrdiff_t>::max());
+        allocated_memory -= size;
+        total_memory -= size;
+        #endif // NETGEN_CHECK_RANGE
+      }
+    }
+
+    int GetId() const { return id; }
+
+    template <typename T1, typename... TRest>
+    void Track( T1 & obj, const std::string& name, TRest & ... rest ) const
+    {
+      Track(obj, name);
+      Track(rest...);
+    }
+
+    template<typename T>
+    void Track( T & obj, const std::string& name ) const
+    {
+      obj.GetMemoryTracer().Activate(obj, name);
+      parents[obj.GetMemoryTracer().GetId()] = id;
+    }
+
+    static std::string GetName(int id)
+    {
+      return names[id];
+    }
+
+    std::string GetName() const
+    {
+      return names[id];
+    }
+
+    template<typename T>
+    void Activate(T& me, const std::string& name) const
+    {
+      if(!id)
+        {
+          const_cast<MemoryTracer*>(this)->id = CreateId(name);
+          if constexpr(detail::has_StartMemoryTracing<T>::value)
+            me.StartMemoryTracing();
+        }
+      else
+        SetName(name);
+    }
+
+    void SetName(const std::string& name) const
+    {
+      names[id] = name;
+    }
+
+
+    static const std::vector<std::string> & GetNames() { return names; }
+    static const std::vector<int> & GetParents() { return parents; }
+    static size_t GetTotalMemory()
+    {
+      #if defined(NETGEN_CHECK_RANGE)
+      return total_memory;
+      #else
+      return 0;
+      #endif // NETGEN_CHECK_RANGE
+    }
+#else // defined(NETGEN_TRACE_MEMORY) && !defined(__CUDA_ARCH__)
+  public:
+    MemoryTracer() {}
+    MemoryTracer( std::string /* name */ ) {}
+    template <typename... TRest>
+    MemoryTracer( std::string /* name */, TRest & ... ) {}
+
+    void Alloc(size_t /* size */) const {}
+    void Free(size_t /* size */) const {}
+    int GetId() const { return 0; }
+
+    template <typename... TRest>
+    void Track(TRest&...) const {}
+
+    static std::string GetName(int /* id */) { return ""; }
+    std::string GetName() const { return ""; }
+    void SetName(std::string /* name */) const {}
+    static size_t GetTotalMemory() { return 0; }
+#endif // NETGEN_TRACE_MEMORY
+  };
+} // namespace ngcore
+
+#endif // NETGEN_CORE_MEMTRACER_HPP

libsrc/core/mpi4py_pycapi.h

@@ -0,0 +1,245 @@
+/* Author:  Lisandro Dalcin   */
+/* Contact: dalcinl@gmail.com */
+
+#ifndef MPI4PY_PYCAPI_H
+#define MPI4PY_PYCAPI_H
+
+#include <mpi.h>
+#include <Python.h>
+
+#define _mpi4py_declare_pycapi(Type, star) \
+static PyTypeObject *_mpi4py_PyMPI##Type = NULL; \
+static PyObject *(*_mpi4py_PyMPI##Type##_New)(MPI_##Type star) = NULL; \
+static MPI_##Type *(*_mpi4py_PyMPI##Type##_Get)(PyObject *) = NULL;
+
+#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE
+_mpi4py_declare_pycapi(Datatype,)
+#define PyMPIDatatype_Type (*_mpi4py_PyMPIDatatype)
+#define PyMPIDatatype_New _mpi4py_PyMPIDatatype_New
+#define PyMPIDatatype_Get _mpi4py_PyMPIDatatype_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_STATUS
+_mpi4py_declare_pycapi(Status,*)
+#define PyMPIStatus_Type (*_mpi4py_PyMPIStatus)
+#define PyMPIStatus_New _mpi4py_PyMPIStatus_New
+#define PyMPIStatus_Get _mpi4py_PyMPIStatus_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST
+_mpi4py_declare_pycapi(Request,)
+#define PyMPIRequest_Type (*_mpi4py_PyMPIRequest)
+#define PyMPIRequest_New _mpi4py_PyMPIRequest_New
+#define PyMPIRequest_Get _mpi4py_PyMPIRequest_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE
+_mpi4py_declare_pycapi(Message,)
+#define PyMPIMessage_Type (*_mpi4py_PyMPIMessage)
+#define PyMPIMessage_New _mpi4py_PyMPIMessage_New
+#define PyMPIMessage_Get _mpi4py_PyMPIMessage_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_OP
+_mpi4py_declare_pycapi(Op,)
+#define PyMPIOp_Type (*_mpi4py_PyMPIOp)
+#define PyMPIOp_New _mpi4py_PyMPIOp_New
+#define PyMPIOp_Get _mpi4py_PyMPIOp_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_GROUP
+_mpi4py_declare_pycapi(Group,)
+#define PyMPIGroup_Type (*_mpi4py_PyMPIGroup)
+#define PyMPIGroup_New _mpi4py_PyMPIGroup_New
+#define PyMPIGroup_Get _mpi4py_PyMPIGroup_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_INFO
+_mpi4py_declare_pycapi(Info,)
+#define PyMPIInfo_Type (*_mpi4py_PyMPIInfo)
+#define PyMPIInfo_New _mpi4py_PyMPIInfo_New
+#define PyMPIInfo_Get _mpi4py_PyMPIInfo_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER
+_mpi4py_declare_pycapi(Errhandler,)
+#define PyMPIErrhandler_Type (*_mpi4py_PyMPIErrhandler)
+#define PyMPIErrhandler_New _mpi4py_PyMPIErrhandler_New
+#define PyMPIErrhandler_Get _mpi4py_PyMPIErrhandler_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_SESSION
+_mpi4py_declare_pycapi(Session,)
+#define PyMPISession_Type (*_mpi4py_PyMPISession)
+#define PyMPISession_New _mpi4py_PyMPISession_New
+#define PyMPISession_Get _mpi4py_PyMPISession_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_COMM
+_mpi4py_declare_pycapi(Comm,)
+#define PyMPIComm_Type (*_mpi4py_PyMPIComm)
+#define PyMPIComm_New _mpi4py_PyMPIComm_New
+#define PyMPIComm_Get _mpi4py_PyMPIComm_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_WIN
+_mpi4py_declare_pycapi(Win,)
+#define PyMPIWin_Type (*_mpi4py_PyMPIWin)
+#define PyMPIWin_New _mpi4py_PyMPIWin_New
+#define PyMPIWin_Get _mpi4py_PyMPIWin_Get
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_FILE
+_mpi4py_declare_pycapi(File,)
+#define PyMPIFile_Type (*_mpi4py_PyMPIFile)
+#define PyMPIFile_New _mpi4py_PyMPIFile_New
+#define PyMPIFile_Get _mpi4py_PyMPIFile_Get
+#endif
+
+#undef _mpi4py_define_pycapi
+
+static int _mpi4py_ImportType(PyObject *module,
+                              const char *type_name,
+                              PyTypeObject **type)
+{
+  PyObject *attr = NULL;
+  attr = PyObject_GetAttrString(module, type_name);
+  if (!attr)
+    goto fn_fail;
+  if (!PyType_Check(attr)) {
+    PyErr_Format(PyExc_TypeError,
+                 "%.200s.%.200s is not a type object",
+                 PyModule_GetName(module), type_name);
+    goto fn_fail;
+  }
+  *type = (PyTypeObject *)attr;
+  return 0;
+ fn_fail:
+  Py_DecRef(attr);
+  return -1;
+}
+
+static int _mpi4py_ImportFunc(PyObject *module,
+                              const char *func_name,
+                              const char *signature,
+                              void (**func)(void))
+{
+    PyObject *pyxcapi = NULL;
+    PyObject *capsule = NULL;
+    union { void *obj; void (*fcn)(void); } ptr;
+    pyxcapi = PyObject_GetAttrString(module, (char *)"__pyx_capi__");
+    if (!pyxcapi)
+      goto fn_fail;
+
+    capsule = PyDict_GetItemString(pyxcapi, func_name);
+    if (!capsule) {
+      PyErr_Format(PyExc_ImportError,
+                   "%.200s does not export expected C function %.200s",
+                   PyModule_GetName(module), func_name);
+      goto fn_fail;
+    }
+    if (!PyCapsule_CheckExact(capsule)) {
+      PyErr_Format(PyExc_TypeError,
+                   "%.200s.%.200s is not a capsule",
+                   PyModule_GetName(module), func_name);
+    }
+    if (!signature) {
+      signature = PyCapsule_GetName(capsule);
+    }
+    if (!PyCapsule_IsValid(capsule, signature)) {
+      PyErr_Format(PyExc_TypeError,
+                   "C function %.200s.%.200s has wrong signature "
+                   "(expected %.500s, got %.500s)",
+                   PyModule_GetName(module), func_name,
+                   signature, PyCapsule_GetName(capsule));
+      goto fn_fail;
+    }
+    ptr.obj = PyCapsule_GetPointer(capsule, signature);
+    if (!ptr.obj)
+      goto fn_fail;
+    *func = ptr.fcn;
+    Py_DecRef(pyxcapi);
+    return 0;
+ fn_fail:
+    Py_DecRef(pyxcapi);
+    return -1;
+}
+
+static int import_mpi4py_MPI(void)
+{
+  PyObject *module = PyImport_ImportModule("mpi4py.MPI");
+  if (!module)
+    goto fn_fail;
+
+#define _mpi4py_import_pycapi(Type) do {                                     \
+    if (_mpi4py_ImportType(module, #Type, &_mpi4py_PyMPI##Type) < 0)         \
+      goto fn_fail;                                                          \
+    if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_New", NULL,               \
+                           (void (**)(void))&_mpi4py_PyMPI##Type##_New) < 0) \
+      goto fn_fail;                                                          \
+    if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_Get", NULL,               \
+                           (void (**)(void))&_mpi4py_PyMPI##Type##_Get) < 0) \
+      goto fn_fail;                                                          \
+  } while (0)
+
+#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE
+  _mpi4py_import_pycapi(Datatype);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_STATUS
+  _mpi4py_import_pycapi(Status);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST
+  _mpi4py_import_pycapi(Request);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE
+  _mpi4py_import_pycapi(Message);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_OP
+  _mpi4py_import_pycapi(Op);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_GROUP
+  _mpi4py_import_pycapi(Group);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_INFO
+  _mpi4py_import_pycapi(Info);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER
+  _mpi4py_import_pycapi(Errhandler);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_SESSION
+  _mpi4py_import_pycapi(Session);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_COMM
+  _mpi4py_import_pycapi(Comm);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_WIN
+  _mpi4py_import_pycapi(Win);
+#endif
+
+#ifndef MPI4PY_LIMITED_API_SKIP_FILE
+  _mpi4py_import_pycapi(File);
+#endif
+
+#undef _mpi4py_import_pycapi
+
+  Py_DecRef(module);
+  return 0;
+ fn_fail:
+  Py_DecRef(module);
+  return -1;
+}
+
+#define __PYX_HAVE_API__mpi4py__MPI
+#define import_mpi4py__MPI import_mpi4py_MPI
+
+#endif /* MPI4PY_PYCAPI_H */

libsrc/core/mpi_wrapper.hpp

@@ -1,13 +1,16 @@
 #ifndef NGCORE_MPIWRAPPER_HPP
 #define NGCORE_MPIWRAPPER_HPP
 
-#ifdef PARALLEL
-#define OMPI_SKIP_MPICXX
-#include <mpi.h>
-#endif
+#include <array>
+
+#include <complex>
 
 #include "array.hpp"
+#include "table.hpp"
 #include "exception.hpp"
+#include "profiler.hpp"
+#include "ngstream.hpp"
+#include "ng_mpi.hpp"
 
 namespace ngcore
 {
@@ -17,37 +20,133 @@ namespace ngcore
   template <class T> struct MPI_typetrait  { };
   
   template <> struct MPI_typetrait<int> {
-    static MPI_Datatype MPIType () { return MPI_INT; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_INT; } };
 
   template <> struct MPI_typetrait<short> {
-    static MPI_Datatype MPIType () { return MPI_SHORT; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_SHORT; } };
 
   template <> struct MPI_typetrait<char> {
-    static MPI_Datatype MPIType () { return MPI_CHAR; } };  
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
 
+  template <> struct MPI_typetrait<signed char> {
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
+  
   template <> struct MPI_typetrait<unsigned char> {
-    static MPI_Datatype MPIType () { return MPI_CHAR; } };  
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
 
   template <> struct MPI_typetrait<size_t> {
-    static MPI_Datatype MPIType () { return MPI_UINT64_T; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_UINT64_T; } };
 
   template <> struct MPI_typetrait<double> {
-    static MPI_Datatype MPIType () { return MPI_DOUBLE; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_DOUBLE; } };
+
+  template <> struct MPI_typetrait<std::complex<double>> {
+    static NG_MPI_Datatype MPIType () { return NG_MPI_CXX_DOUBLE_COMPLEX; } };
 
   template <> struct MPI_typetrait<bool> {
-    static MPI_Datatype MPIType () { return MPI_C_BOOL; } };
+    static NG_MPI_Datatype MPIType () { return NG_MPI_C_BOOL; } };
 
+
+  template<typename T, size_t S>
+  struct MPI_typetrait<std::array<T,S>>
+  {
+    static NG_MPI_Datatype MPIType ()
+    { 
+      static NG_MPI_Datatype NG_MPI_T = 0;
+      if (!NG_MPI_T)
+	{
+	  NG_MPI_Type_contiguous ( S, MPI_typetrait<T>::MPIType(), &NG_MPI_T);
+	  NG_MPI_Type_commit ( &NG_MPI_T );
+	}
+      return NG_MPI_T;
+    }
+  };
   
   template <class T, class T2 = decltype(MPI_typetrait<T>::MPIType())>
-  inline MPI_Datatype GetMPIType () {
+  inline NG_MPI_Datatype GetMPIType () {
     return MPI_typetrait<T>::MPIType();
   }
+
+  template <class T>
+  inline NG_MPI_Datatype GetMPIType (T &) {
+    return GetMPIType<T>();
+  }
+
+  class NgMPI_Request
+  {
+    NG_MPI_Request request;
+  public:
+    NgMPI_Request (NG_MPI_Request requ) : request{requ} { }
+    NgMPI_Request (const NgMPI_Request&) = delete;    
+    NgMPI_Request (NgMPI_Request&&) = default;
+    ~NgMPI_Request () { NG_MPI_Wait (&request, NG_MPI_STATUS_IGNORE); }
+    void Wait() {  NG_MPI_Wait (&request, NG_MPI_STATUS_IGNORE); }
+    operator NG_MPI_Request() &&
+    {
+      auto tmp = request;
+      request = NG_MPI_REQUEST_NULL;
+      return tmp;
+    }
+  };
+
+  class NgMPI_Requests
+  {
+    Array<NG_MPI_Request> requests;
+  public:
+    NgMPI_Requests() = default;
+    ~NgMPI_Requests() { WaitAll(); }
+
+    void Reset() { requests.SetSize0(); }
+    
+    NgMPI_Requests & operator+= (NgMPI_Request && r)
+    {
+      requests += NG_MPI_Request(std::move(r));
+      return *this;
+    }
+
+    NgMPI_Requests & operator+= (NG_MPI_Request r)
+    {
+      requests += r;
+      return *this;
+    }
+    
+    void WaitAll()
+    {
+      static Timer t("NgMPI - WaitAll"); RegionTimer reg(t);    
+      if (!requests.Size()) return;
+      NG_MPI_Waitall (requests.Size(), requests.Data(), NG_MPI_STATUSES_IGNORE);
+    }
+
+    int WaitAny ()
+    {
+      int nr;
+      NG_MPI_Waitany (requests.Size(), requests.Data(), &nr, NG_MPI_STATUS_IGNORE);
+      return nr;
+    }
+  };
+  
+  [[deprecated("use requests.WaitAll instread")]]
+  inline void MyMPI_WaitAll (FlatArray<NG_MPI_Request> requests)
+  {
+    static Timer t("MPI - WaitAll"); RegionTimer reg(t);    
+    if (!requests.Size()) return;
+    NG_MPI_Waitall (requests.Size(), requests.Data(), NG_MPI_STATUSES_IGNORE);
+  }
+
+  [[deprecated("use requests.WaitAny instread")]]  
+  inline int MyMPI_WaitAny (FlatArray<NG_MPI_Request> requests)
+  {
+    int nr;
+    NG_MPI_Waitany (requests.Size(), requests.Data(), &nr, NG_MPI_STATUS_IGNORE);
+    return nr;
+  }
+
   
 
   class NgMPI_Comm
   {
   protected:
-    MPI_Comm comm;
+    NG_MPI_Comm comm;
     bool valid_comm;
     int * refcount;
     int rank, size;
@@ -56,16 +155,27 @@ namespace ngcore
       : valid_comm(false), refcount(nullptr), rank(0), size(1)
     { ; }
 
-    NgMPI_Comm (MPI_Comm _comm, bool owns = false)
+    NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false)
       : comm(_comm), valid_comm(true)
     {
+      int flag;
+      NG_MPI_Initialized (&flag);
+      if (!flag)
+        {
+          valid_comm = false;
+          refcount = nullptr;
+          rank = 0;
+          size = 1;
+          return;
+        }
+
       if (!owns)
         refcount = nullptr;
       else
         refcount = new int{1};
       
-      MPI_Comm_rank(comm, &rank);
-      MPI_Comm_size(comm, &size);
+      NG_MPI_Comm_rank(comm, &rank);
+      NG_MPI_Comm_size(comm, &size);
     }
     
     NgMPI_Comm (const NgMPI_Comm & c)
@@ -86,14 +196,19 @@ namespace ngcore
     {
       if (refcount)
         if (--(*refcount) == 0)
-          MPI_Comm_free(&comm);
+          NG_MPI_Comm_free(&comm);
     }
 
+    bool ValidCommunicator() const
+    {
+      return valid_comm;
+    }
+    
     NgMPI_Comm & operator= (const NgMPI_Comm & c)
     {
       if (refcount)
         if (--(*refcount) == 0)
-          MPI_Comm_free(&comm);
+          NG_MPI_Comm_free(&comm);
 
       refcount = c.refcount;
       if (refcount) (*refcount)++;      
@@ -109,7 +224,7 @@ namespace ngcore
       InvalidCommException() : Exception("Do not have a valid communicator") { ; }
     };
     
-    operator MPI_Comm() const {
+    operator NG_MPI_Comm() const {
       if (!valid_comm) throw InvalidCommException();
       return comm;
     }
@@ -117,7 +232,8 @@ namespace ngcore
     int Rank() const { return rank; }
     int Size() const { return size; }
     void Barrier() const {
-      if (size > 1) MPI_Barrier (comm);
+      static Timer t("MPI - Barrier"); RegionTimer reg(t);
+      if (size > 1) NG_MPI_Barrier (comm);
     }
     
 
@@ -125,67 +241,82 @@ namespace ngcore
 
     template<typename T, typename T2 = decltype(GetMPIType<T>())>
     void Send (T & val, int dest, int tag) const {
-      MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
+      NG_MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
+    }
+
+    void Send (const std::string & s, int dest, int tag) const {
+      NG_MPI_Send( const_cast<char*> (&s[0]), s.length(), NG_MPI_CHAR, dest, tag, comm);
     }
     
-    template<typename T, typename T2 = decltype(GetMPIType<T>())>
-    void Send(FlatArray<T> s, int dest, int tag) const {
-      MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
+    template<typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
+    void Send(FlatArray<T,TI> s, int dest, int tag) const {
+      NG_MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
     void Recv (T & val, int src, int tag) const {
-      MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, MPI_STATUS_IGNORE);
+      NG_MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, NG_MPI_STATUS_IGNORE);
     }
 
-    template <typename T, typename T2 = decltype(GetMPIType<T>())>
-    void Recv (FlatArray <T> s, int src, int tag) const {
-      MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, MPI_STATUS_IGNORE);
+    void Recv (std::string & s, int src, int tag) const {    
+      NG_MPI_Status status;
+      int len;
+      NG_MPI_Probe (src, tag, comm, &status);
+      NG_MPI_Get_count (&status, NG_MPI_CHAR, &len);
+      // s.assign (len, ' ');
+      s.resize (len);
+      NG_MPI_Recv( &s[0], len, NG_MPI_CHAR, src, tag, comm, NG_MPI_STATUS_IGNORE);
     }
     
-    template <typename T, typename T2 = decltype(GetMPIType<T>())>
-    void Recv (Array <T> & s, int src, int tag) const
+
+    template <typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
+    void Recv (FlatArray <T,TI> s, int src, int tag) const {
+      NG_MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, NG_MPI_STATUS_IGNORE);
+    }
+    
+    template <typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
+    void Recv (Array <T,TI> & s, int src, int tag) const
     {
-      MPI_Status status;
+      NG_MPI_Status status;
       int len;
-      const MPI_Datatype MPI_T  = GetMPIType<T> ();
-      MPI_Probe (src, tag, comm, &status);
-      MPI_Get_count (&status, MPI_T, &len);
+      const NG_MPI_Datatype NG_MPI_T  = GetMPIType<T> ();
+      NG_MPI_Probe (src, tag, comm, &status);
+      NG_MPI_Get_count (&status, NG_MPI_T, &len);
       s.SetSize (len);
-      MPI_Recv (s.Data(), len, MPI_T, src, tag, comm, MPI_STATUS_IGNORE);
+      NG_MPI_Recv (s.Data(), len, NG_MPI_T, src, tag, comm, NG_MPI_STATUS_IGNORE);
     }
 
     /** --- non-blocking P2P --- **/
-    
-    template<typename T, typename T2 = decltype(GetMPIType<T>())> 
-    MPI_Request ISend (T & val, int dest, int tag) const
-    {
-      MPI_Request request;
-      MPI_Isend (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
-      return request;
-    }
-
-    template<typename T, typename T2 = decltype(GetMPIType<T>())>
-    MPI_Request ISend (const FlatArray<T> & s, int dest, int tag) const
-    {
-      MPI_Request request;
-      MPI_Isend (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
-      return request;
-    }
 
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
-    MPI_Request IRecv (T & val, int dest, int tag) const
+    [[nodiscard]] NG_MPI_Request ISend (T & val, int dest, int tag) const
     {
-      MPI_Request request;
-      MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
+      NG_MPI_Request request;
+      NG_MPI_Isend (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
       return request;
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())>
-    MPI_Request IRecv (const FlatArray<T> & s, int src, int tag) const
+    [[nodiscard]] NG_MPI_Request ISend (FlatArray<T> s, int dest, int tag) const
+    {
+      NG_MPI_Request request;
+      NG_MPI_Isend (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
+      return request;
+    }
+    
+    template<typename T, typename T2 = decltype(GetMPIType<T>())> 
+    [[nodiscard]] NG_MPI_Request IRecv (T & val, int dest, int tag) const
+    {
+      NG_MPI_Request request;
+      NG_MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
+      return request;
+    }
+    
+    template<typename T, typename T2 = decltype(GetMPIType<T>())>
+    [[nodiscard]] NG_MPI_Request IRecv (FlatArray<T> s, int src, int tag) const
     { 
-      MPI_Request request;
-      MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
+      NG_MPI_Request request;
+      NG_MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
       return request;
     }
 
@@ -193,91 +324,247 @@ namespace ngcore
     /** --- collectives --- **/
 
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
-    T Reduce (T d, const MPI_Op & op, int root = 0) const
+    T Reduce (T d, const NG_MPI_Op & op, int root = 0) const
     {
+      static Timer t("MPI - Reduce"); RegionTimer reg(t);          
       if (size == 1) return d;
       
       T global_d;
-      MPI_Reduce (&d, &global_d, 1, GetMPIType<T>(), op, root, comm);
+      NG_MPI_Reduce (&d, &global_d, 1, GetMPIType<T>(), op, root, comm);
       return global_d;
     }
     
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
-    T AllReduce (T d, const MPI_Op & op) const
+    T AllReduce (T d, const NG_MPI_Op & op) const
     {
+      static Timer t("MPI - AllReduce"); RegionTimer reg(t);
       if (size == 1) return d;
       
       T global_d;
-      MPI_Allreduce ( &d, &global_d, 1, GetMPIType<T>(), op, comm);
+      NG_MPI_Allreduce ( &d, &global_d, 1, GetMPIType<T>(), op, comm);
       return global_d;
     }
 
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
-    void Bcast (T & s, int root = 0) const {
-      if (size == 1) return ;
-      MPI_Bcast (&s, 1, GetMPIType<T>(), root, comm);
+    void AllReduce (FlatArray<T> d, const NG_MPI_Op & op) const
+    {
+      static Timer t("MPI - AllReduce Array"); RegionTimer reg(t);
+      if (size == 1) return;
+      
+      NG_MPI_Allreduce (NG_MPI_IN_PLACE, d.Data(), d.Size(), GetMPIType<T>(), op, comm);
     }
     
+    template <typename T, typename T2 = decltype(GetMPIType<T>())> 
+    void Bcast (T & s, int root = 0) const {
+      if (size == 1) return;
+      static Timer t("MPI - Bcast"); RegionTimer reg(t);
+      NG_MPI_Bcast (&s, 1, GetMPIType<T>(), root, comm);
+    }
+
+
+    template <class T, size_t S>
+    void Bcast (std::array<T,S> & d, int root = 0) const
+    {
+      if (size == 1) return;
+      if (S != 0)
+        NG_MPI_Bcast (&d[0], S, GetMPIType<T>(), root, comm);
+    }
+    
+    
+    template <class T>
+    void Bcast (Array<T> & d, int root = 0) const
+    {
+      if (size == 1) return;
+      
+      int ds = d.Size();
+      Bcast (ds, root);
+      if (Rank() != root) d.SetSize (ds);
+      if (ds != 0)
+        NG_MPI_Bcast (d.Data(), ds, GetMPIType<T>(), root, comm);
+    }
+
+    
     void Bcast (std::string & s, int root = 0) const 
     {
       if (size == 1) return;
       int len = s.length();
       Bcast (len, root);
       if (rank != 0) s.resize (len);
-      MPI_Bcast (&s[0], len, MPI_CHAR, root, comm);
+      NG_MPI_Bcast (&s[0], len, NG_MPI_CHAR, root, comm);
     }
 
+
+    
+    template <class T, size_t S>
+    [[nodiscard]] NgMPI_Request IBcast (std::array<T,S> & d, int root = 0) const
+    {
+      NG_MPI_Request request;      
+      NG_MPI_Ibcast (&d[0], S, GetMPIType<T>(), root, comm, &request);
+      return request;
+    }
+
+    template <class T>
+     [[nodiscard]] NgMPI_Request IBcast (FlatArray<T> d, int root = 0) const
+    {
+      NG_MPI_Request request;      
+      int ds = d.Size();
+      NG_MPI_Ibcast (d.Data(), ds, GetMPIType<T>(), root, comm, &request);
+      return request;
+    }
+
+    
+    template <typename T>
+    void AllToAll (FlatArray<T> send, FlatArray<T> recv) const
+    {
+      NG_MPI_Alltoall (send.Data(), 1, GetMPIType<T>(),
+                       recv.Data(), 1, GetMPIType<T>(), comm);
+    }
+
+
+    template <typename T>
+    void ScatterRoot (FlatArray<T> send) const
+    {
+      if (size == 1) return;
+      NG_MPI_Scatter (send.Data(), 1, GetMPIType<T>(),
+                      NG_MPI_IN_PLACE, -1, GetMPIType<T>(), 0, comm);
+    }
+    
+    template <typename T>
+    void Scatter (T & recv) const
+    {
+      if (size == 1) return;      
+      NG_MPI_Scatter (NULL, 0, GetMPIType<T>(),
+                      &recv, 1, GetMPIType<T>(), 0, comm);
+    }
+
+    template <typename T>
+    void GatherRoot (FlatArray<T> recv) const
+    {
+      recv[0] = T(0);
+      if (size == 1) return;      
+      NG_MPI_Gather (NG_MPI_IN_PLACE, 1, GetMPIType<T>(),
+                     recv.Data(), 1, GetMPIType<T>(), 0, comm);
+    }
+
+    template <typename T>
+    void Gather (T send) const
+    {
+      if (size == 1) return;            
+      NG_MPI_Gather (&send, 1, GetMPIType<T>(),
+                  NULL, 1, GetMPIType<T>(), 0, comm);
+    }
+
+    
+    template <typename T>
+    void AllGather (T val, FlatArray<T> recv) const
+    {
+      if (size == 1)
+        {
+          recv[0] = val;
+          return;
+        }
+      NG_MPI_Allgather (&val, 1, GetMPIType<T>(),
+                     recv.Data(), 1, GetMPIType<T>(), 
+                     comm);
+    }
+    
+
+
+    template <typename T>
+    void ExchangeTable (DynamicTable<T> & send_data, 
+                        DynamicTable<T> & recv_data, int tag)
+    {
+      Array<int> send_sizes(size);
+      Array<int> recv_sizes(size);
+      
+      for (int i = 0; i < size; i++)
+        send_sizes[i] = send_data[i].Size();
+      
+      AllToAll (send_sizes, recv_sizes);
+    
+      recv_data = DynamicTable<T> (recv_sizes, true);
+      
+      NgMPI_Requests requests;
+      for (int dest = 0; dest < size; dest++)
+        if (dest != rank && send_data[dest].Size())
+          requests += ISend (FlatArray<T>(send_data[dest]), dest, tag);
+      
+      for (int dest = 0; dest < size; dest++)
+        if (dest != rank && recv_data[dest].Size())
+          requests += IRecv (FlatArray<T>(recv_data[dest]), dest, tag);
+
+      requests.WaitAll();
+    }
+    
+
+
+
+    
     NgMPI_Comm SubCommunicator (FlatArray<int> procs) const
     {
-      MPI_Comm subcomm;
-      MPI_Group gcomm, gsubcomm;
-      MPI_Comm_group(comm, &gcomm);
-      MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm);
-      MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm);
+      NG_MPI_Comm subcomm;
+      NG_MPI_Group gcomm, gsubcomm;
+      NG_MPI_Comm_group(comm, &gcomm);
+      NG_MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm);
+      NG_MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm);
       return NgMPI_Comm(subcomm, true);
     }
 
   }; // class NgMPI_Comm
 
-  NETGEN_INLINE void MyMPI_WaitAll (FlatArray<MPI_Request> requests)
-  {
-    if (!requests.Size()) return;
-    MPI_Waitall (requests.Size(), requests.Data(), MPI_STATUSES_IGNORE);
-  }
-  
-  NETGEN_INLINE int MyMPI_WaitAny (FlatArray<MPI_Request> requests)
-  {
-    int nr;
-    MPI_Waitany (requests.Size(), requests.Data(), &nr, MPI_STATUS_IGNORE);
-    return nr;
-  }
-
 #else // PARALLEL
-  class MPI_Comm {
+  class NG_MPI_Comm {
     int nr;
   public:
-    MPI_Comm (int _nr = 0) : nr(_nr) { ; }
+    NG_MPI_Comm (int _nr = 0) : nr(_nr) { ; }
     operator int() const { return nr; }
-    bool operator== (MPI_Comm c2) const { return nr == c2.nr; }
+    bool operator== (NG_MPI_Comm c2) const { return nr == c2.nr; }
   };
-  static MPI_Comm MPI_COMM_WORLD = 12345, MPI_COMM_NULL = 10000;
+  static NG_MPI_Comm NG_MPI_COMM_WORLD = 12345, NG_MPI_COMM_NULL = 10000;
 
-  typedef int MPI_Op;
-  typedef int MPI_Request;
+  typedef int NG_MPI_Op;
+  typedef int NG_MPI_Datatype;  
+  typedef int NG_MPI_Request;
   
-  enum { MPI_SUM = 0, MPI_MIN = 1, MPI_MAX = 2 };
+  enum { NG_MPI_SUM = 0, NG_MPI_MIN = 1, NG_MPI_MAX = 2, NG_MPI_LOR = 4711 };
+
+  inline void NG_MPI_Type_contiguous ( int, NG_MPI_Datatype, NG_MPI_Datatype*) { ; } 
+  inline void NG_MPI_Type_commit ( NG_MPI_Datatype * ) { ; }
+
+  template <class T> struct MPI_typetrait  {
+    static NG_MPI_Datatype MPIType () { return -1; }    
+  };
+  template <class T, class T2=void>
+  inline NG_MPI_Datatype GetMPIType () { return -1; }
+
+  class NgMPI_Request {
+  public:
+    NgMPI_Request() = default;
+    NgMPI_Request(NgMPI_Request &&) { ; }    
+    NgMPI_Request(NG_MPI_Request &&) { ; }
+  };
+  class NgMPI_Requests
+  {
+  public:
+    NgMPI_Requests & operator+= (NgMPI_Request &&) { return *this; }
+    NgMPI_Requests & operator+= (NG_MPI_Request r) { return *this; }
+    void Reset() { ; }
+    void WaitAll() { ; }
+    int WaitAny() { return 0; }
+  };
   
   class NgMPI_Comm
   {
     
   public:
     NgMPI_Comm () { ; } 
-    NgMPI_Comm (MPI_Comm _comm, bool owns = false) { ; }
+    NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false) { ; }
 
     size_t Rank() const { return 0; }
     size_t Size() const { return 1; }
+    bool ValidCommunicator() const { return false; }
     void Barrier() const { ; } 
-    operator MPI_Comm() const { return MPI_Comm(); }
+    operator NG_MPI_Comm() const { return NG_MPI_Comm(); }
 
     template<typename T>
     void Send( T & val, int dest, int tag) const { ; }
@@ -295,31 +582,58 @@ namespace ngcore
     void Recv (Array <T> & s, int src, int tag) const { ; }
 
     template<typename T>
-    MPI_Request ISend (T & val, int dest, int tag) const { return 0; } 
+    NG_MPI_Request ISend (T & val, int dest, int tag) const { return 0; } 
     
     template<typename T>
-    MPI_Request ISend (const FlatArray<T> & s, int dest, int tag) const { return 0; }
+    NG_MPI_Request ISend (FlatArray<T> s, int dest, int tag) const { return 0; }
 
     template<typename T>
-    MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } 
+    NG_MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } 
     
     template<typename T>
-    MPI_Request IRecv (const FlatArray<T> & s, int src, int tag) const { return 0; }
+    NG_MPI_Request IRecv (FlatArray<T> s, int src, int tag) const { return 0; }
 
     template <typename T>
-    T Reduce (T d, const MPI_Op & op, int root = 0) const { return d; }
+    T Reduce (T d, const NG_MPI_Op & op, int root = 0) const { return d; }
     
     template <typename T>
-    T AllReduce (T d, const MPI_Op & op) const { return d; }
+    T AllReduce (T d, const NG_MPI_Op & op) const { return d; }
 
+    template <typename T>
+    void AllReduce (FlatArray<T> d, const NG_MPI_Op & op) const { ; }
+    
     template <typename T>
     void Bcast (T & s, int root = 0) const { ; } 
 
+    template <class T, size_t S>
+    void Bcast (std::array<T,S> & d, int root = 0) const {}
+    
+    template <class T>
+    void Bcast (Array<T> & d, int root = 0) const { ; } 
+
+    template <class T, size_t S>
+    NG_MPI_Request IBcast (std::array<T,S> & d, int root = 0) const { return 0; }
+
+    template <class T>
+    NG_MPI_Request IBcast (FlatArray<T> d, int root = 0) const { return 0; } 
+    
+    template <typename T>
+    void AllGather (T val, FlatArray<T> recv) const
+    {
+      recv[0] = val;
+    }
+
+    template <typename T>
+    void ExchangeTable (DynamicTable<T> & send_data, 
+                        DynamicTable<T> & recv_data, int tag) { ; }
+
+    
     NgMPI_Comm SubCommunicator (FlatArray<int> procs) const
     { return *this; }
   };  
 
-  NETGEN_INLINE void MyMPI_WaitAll (FlatArray<MPI_Request> requests) { ; }
+  inline void MyMPI_WaitAll (FlatArray<NG_MPI_Request> requests) { ; }
+  inline int MyMPI_WaitAny (FlatArray<NG_MPI_Request> requests) { return 0; }
 
 #endif // PARALLEL
 

libsrc/core/ng_mpi.cpp

@@ -0,0 +1,184 @@
+#define OMPI_SKIP_MPICXX
+#include <mpi.h>
+
+#include "ng_mpi.hpp"
+
+#include <type_traits>
+
+#include "array.hpp"
+#include "ngcore_api.hpp"
+
+#ifdef NG_PYTHON
+#include "pybind11/pytypes.h"
+#include "python_ngcore.hpp"
+
+#define MPI4PY_LIMITED_API 1
+#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1
+#define MPI4PY_LIMITED_API_SKIP_SESSION 1
+#include "mpi4py_pycapi.h"  // mpi4py < 4.0.0
+#endif
+
+#ifdef MSMPI_VER
+int MPI_Comm_create_group(MPI_Comm arg0, MPI_Group arg1, int arg2,
+                          MPI_Comm* arg3) {
+  throw std::runtime_error(
+      "MPI_Comm_create_group not supported on Microsoft MPI");
+}
+static MPI_Datatype MPI_CXX_DOUBLE_COMPLEX;
+#endif  // MSMPI_VER
+
+namespace ngcore {
+
+static_assert(sizeof(MPI_Status) <= sizeof(NG_MPI_Status), "Size mismatch");
+static_assert(alignof(MPI_Status) <= alignof(NG_MPI_Status), "Size mismatch");
+
+int mpi2ng(int value) { return value; }
+void* mpi2ng(void* ptr) { return ptr; }
+
+NG_MPI_Status* mpi2ng(MPI_Status* status) {
+  return reinterpret_cast<NG_MPI_Status*>(status);
+}
+
+#if !defined(MPICH) && !defined(MSMPI_VER)
+NG_MPI_Comm mpi2ng(MPI_Comm comm) { return reinterpret_cast<uintptr_t>(comm); }
+#endif
+
+template <size_t size, size_t stride>
+void gather_strided_array(size_t count, char* data) {
+  static_assert(size <= stride, "Size must be less than or equal to stride");
+  if constexpr (size < stride) {
+    char* dst = data;
+    char* src = data;
+    for ( [[maybe_unused]] auto i : Range(count)) {
+      memcpy(dst, src, size);
+      dst += size;
+      src += stride;
+    }
+  }
+}
+
+template <typename T>
+T cast_ng2mpi(uintptr_t obj) {
+  if constexpr (std::is_pointer_v<T>)
+    return reinterpret_cast<T>(obj);
+  else
+    return static_cast<T>(obj);
+}
+
+template <typename T>
+T cast_ng2mpi(uintptr_t* ptr) {
+  if constexpr (std::is_pointer_v<T>)
+    return reinterpret_cast<T>(ptr);
+  else
+    return static_cast<T>(ptr);
+}
+
+template <typename T, typename TSrc>
+T* cast_ng2mpi(TSrc* ptr, int count) {
+  gather_strided_array<sizeof(T), sizeof(TSrc)>(count,
+                                                reinterpret_cast<char*>(ptr));
+  return reinterpret_cast<T*>(ptr);
+}
+
+MPI_Comm ng2mpi(NG_MPI_Comm comm) {
+  static_assert(sizeof(MPI_Comm) <= sizeof(comm.value), "Size mismatch");
+  static_assert(alignof(MPI_Comm) <= alignof(NG_MPI_Comm), "Size mismatch");
+  return cast_ng2mpi<MPI_Comm>(comm.value);
+}
+
+MPI_Group ng2mpi(NG_MPI_Group group) {
+  static_assert(sizeof(MPI_Group) <= sizeof(group.value), "Size mismatch");
+  static_assert(alignof(MPI_Group) <= alignof(NG_MPI_Group), "Size mismatch");
+  return cast_ng2mpi<MPI_Group>(group.value);
+}
+
+MPI_Comm* ng2mpi(NG_MPI_Comm* comm) {
+  return cast_ng2mpi<MPI_Comm*>(&comm->value);
+}
+MPI_Group* ng2mpi(NG_MPI_Group* group) {
+  return cast_ng2mpi<MPI_Group*>(&group->value);
+}
+MPI_Datatype* ng2mpi(NG_MPI_Datatype* type) {
+  return cast_ng2mpi<MPI_Datatype*>(&type->value);
+}
+MPI_Datatype* ng2mpi(NG_MPI_Datatype* type, int count) {
+  return cast_ng2mpi<MPI_Datatype>(&type->value, count);
+}
+MPI_Request* ng2mpi(NG_MPI_Request* request) {
+  return cast_ng2mpi<MPI_Request*>(&request->value);
+}
+MPI_Request* ng2mpi(NG_MPI_Request* request, int count) {
+  return cast_ng2mpi<MPI_Request>(&request->value, count);
+}
+MPI_Status* ng2mpi(NG_MPI_Status* status) {
+  return reinterpret_cast<MPI_Status*>(status);
+}
+MPI_Aint* ng2mpi(NG_MPI_Aint* aint) {
+  return reinterpret_cast<MPI_Aint*>(aint);
+}
+MPI_Aint* ng2mpi(NG_MPI_Aint* aint, int count) {
+  return cast_ng2mpi<MPI_Aint>(aint, count);
+}
+
+MPI_Datatype ng2mpi(NG_MPI_Datatype type) {
+  static_assert(sizeof(MPI_Datatype) <= sizeof(type.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Datatype>(type.value);
+}
+
+MPI_Request ng2mpi(NG_MPI_Request request) {
+  static_assert(sizeof(MPI_Request) <= sizeof(request.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Request>(request.value);
+}
+
+MPI_Op ng2mpi(NG_MPI_Op op) {
+  static_assert(sizeof(MPI_Op) <= sizeof(op.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Op>(op.value);
+}
+
+MPI_Aint ng2mpi(NG_MPI_Aint aint) {
+  static_assert(sizeof(MPI_Aint) <= sizeof(aint.value), "Size mismatch");
+  return cast_ng2mpi<MPI_Aint>(aint.value);
+}
+
+void* ng2mpi(void* ptr) { return ptr; }
+char* ng2mpi(char* ptr) { return ptr; }
+char*** ng2mpi(char*** ptr) { return ptr; }
+int* ng2mpi(int* ptr) { return ptr; }
+int ng2mpi(int value) { return value; }
+
+}  // namespace ngcore
+
+using namespace ngcore;
+
+extern "C" {
+NGCORE_API_EXPORT void ng_init_mpi();
+}
+
+static bool imported_mpi4py = false;
+
+void ng_init_mpi() {
+#ifdef NG_PYTHON
+  NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool {
+    if (!imported_mpi4py) {
+      import_mpi4py__MPI();
+      imported_mpi4py = true;
+    }
+    PyObject* py_src = src.ptr();
+    [[maybe_unused]] auto type = Py_TYPE(py_src);
+    if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
+      dst = mpi2ng(*PyMPIComm_Get(py_src));
+      return !PyErr_Occurred();
+    }
+    return false;
+  };
+  NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle {
+    if (!imported_mpi4py) {
+      import_mpi4py__MPI();
+      imported_mpi4py = true;
+    }
+    return py::handle(PyMPIComm_New(ng2mpi(src)));
+  };
+#endif  // NG_PYTHON
+
+#include "ng_mpi_generated_init.hpp"
+}

libsrc/core/ng_mpi.hpp

@@ -0,0 +1,94 @@
+#ifndef NG_MPI_HPP_INCLUDED
+#define NG_MPI_HPP_INCLUDED
+
+#ifdef PARALLEL
+
+#include <cstdint>
+#include <filesystem>
+#include <optional>
+
+#include "ngcore_api.hpp"
+
+#ifndef NG_MPI_WRAPPER
+#define OMPI_SKIP_MPICXX
+#include <mpi.h>
+#endif  // NG_MPI_WRAPPER
+
+namespace ngcore {
+
+NGCORE_API bool MPI_Loaded();
+NGCORE_API void InitMPI(
+    std::optional<std::filesystem::path> mpi_lib_path = std::nullopt);
+
+#ifdef NG_MPI_WRAPPER
+inline void not_implemented() { throw std::runtime_error("Not implemented"); }
+
+struct NG_MPI_Status {
+  uintptr_t data[4];
+};
+
+struct NG_MPI_Comm {
+  uintptr_t value;
+  NG_MPI_Comm() { value = 0; }
+  NG_MPI_Comm(uintptr_t value_) : value(value_) {}
+  NG_MPI_Comm(const NG_MPI_Comm &comm) : value(comm.value) {}
+
+  void operator=(int value_) { value = value_; }
+  void operator=(uintptr_t value_) { value = value_; }
+  bool operator==(const NG_MPI_Comm &comm) const { return value == comm.value; }
+  bool operator!=(const NG_MPI_Comm &comm) const { return value != comm.value; }
+};
+
+struct NG_MPI_Datatype {
+  uintptr_t value = 0;
+  NG_MPI_Datatype() = default;
+  NG_MPI_Datatype(uintptr_t value_) : value(value_) {}
+  operator bool() const { return value != 0; }
+  void operator=(NG_MPI_Datatype type) { value = type.value; }
+  void operator=(uintptr_t value_) { value = value_; }
+  void operator=(void *value_) { value = reinterpret_cast<uintptr_t>(value_); }
+};
+
+struct NG_MPI_Request {
+  uintptr_t value = 0;
+  NG_MPI_Request() = default;
+  NG_MPI_Request(uintptr_t value_) : value(value_) {}
+  void operator=(uintptr_t value_) { value = value_; }
+  void operator=(void *value_) { value = reinterpret_cast<uintptr_t>(value_); }
+};
+
+struct NG_MPI_Op {
+  uintptr_t value;
+  NG_MPI_Op(uintptr_t value_) : value(value_) {}
+  void operator=(uintptr_t value_) { value = value_; }
+  void operator=(void *value_) { value = reinterpret_cast<uintptr_t>(value_); }
+};
+
+struct NG_MPI_Group {
+  uintptr_t value = 0;
+  NG_MPI_Group(uintptr_t value_) : value(value_) {}
+  NG_MPI_Group() = default;
+};
+
+struct NG_MPI_Aint {
+  intptr_t value = 0;
+  NG_MPI_Aint(intptr_t value_) : value(value_) {}
+  NG_MPI_Aint() = default;
+};
+
+#else   // NG_MPI_WRAPPER
+using NG_MPI_Comm = MPI_Comm;
+using NG_MPI_Status = MPI_Status;
+using NG_MPI_Datatype = MPI_Datatype;
+using NG_MPI_Request = MPI_Request;
+using NG_MPI_Op = MPI_Op;
+using NG_MPI_Group = MPI_Group;
+using NG_MPI_Aint = MPI_Aint;
+#endif  // NG_MPI_WRAPPER
+
+#include "ng_mpi_generated_declarations.hpp"
+
+}  // namespace ngcore
+
+#endif  // PARALLEL
+#endif  // NG_MPI_HPP_INCLUDED

libsrc/core/ng_mpi_generated_declarations.hpp

@@ -0,0 +1,155 @@
+#ifdef NG_MPI_WRAPPER
+NGCORE_API extern double (*NG_MPI_Wtime)();
+NGCORE_API extern int (*NG_MPI_Allgather)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Allreduce)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Alltoall)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Barrier)(NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Bcast)(void*, int, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Ibcast)(void*, int, NG_MPI_Datatype, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Comm_c2f)(NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Comm_create)(NG_MPI_Comm, NG_MPI_Group, NG_MPI_Comm*);
+NGCORE_API extern int (*NG_MPI_Comm_create_group)(NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*);
+NGCORE_API extern int (*NG_MPI_Comm_free)(NG_MPI_Comm*);
+NGCORE_API extern int (*NG_MPI_Comm_group)(NG_MPI_Comm, NG_MPI_Group*);
+NGCORE_API extern int (*NG_MPI_Comm_rank)(NG_MPI_Comm, int*);
+NGCORE_API extern int (*NG_MPI_Comm_size)(NG_MPI_Comm, int*);
+NGCORE_API extern int (*NG_MPI_Finalize)();
+NGCORE_API extern int (*NG_MPI_Gather)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Gatherv)(void*, int, NG_MPI_Datatype, void*, int*, int*, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Get_count)(NG_MPI_Status*, NG_MPI_Datatype, int*);
+NGCORE_API extern int (*NG_MPI_Get_processor_name)(char*, int*);
+NGCORE_API extern int (*NG_MPI_Group_incl)(NG_MPI_Group, int, int*, NG_MPI_Group*);
+NGCORE_API extern int (*NG_MPI_Init)(int*, char***);
+NGCORE_API extern int (*NG_MPI_Init_thread)(int*, char***, int, int*);
+NGCORE_API extern int (*NG_MPI_Initialized)(int*);
+NGCORE_API extern int (*NG_MPI_Iprobe)(int, int, NG_MPI_Comm, int*, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Irecv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Isend)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Probe)(int, int, NG_MPI_Comm, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Query_thread)(int*);
+NGCORE_API extern int (*NG_MPI_Recv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Recv_init)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Reduce)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Reduce_local)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op);
+NGCORE_API extern int (*NG_MPI_Request_free)(NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Scatter)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Send)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm);
+NGCORE_API extern int (*NG_MPI_Send_init)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Startall)(int, NG_MPI_Request*);
+NGCORE_API extern int (*NG_MPI_Type_commit)(NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_contiguous)(int, NG_MPI_Datatype, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_create_resized)(NG_MPI_Datatype, NG_MPI_Aint, NG_MPI_Aint, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_create_struct)(int, int*, NG_MPI_Aint*, NG_MPI_Datatype*, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_free)(NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_get_extent)(NG_MPI_Datatype, NG_MPI_Aint*, NG_MPI_Aint*);
+NGCORE_API extern int (*NG_MPI_Type_indexed)(int, int*, int*, NG_MPI_Datatype, NG_MPI_Datatype*);
+NGCORE_API extern int (*NG_MPI_Type_size)(NG_MPI_Datatype, int*);
+NGCORE_API extern int (*NG_MPI_Wait)(NG_MPI_Request*, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Waitall)(int, NG_MPI_Request*, NG_MPI_Status*);
+NGCORE_API extern int (*NG_MPI_Waitany)(int, NG_MPI_Request*, int*, NG_MPI_Status*);
+NGCORE_API extern NG_MPI_Comm NG_MPI_COMM_NULL;
+NGCORE_API extern NG_MPI_Comm NG_MPI_COMM_WORLD;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_CHAR;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_C_BOOL;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_DATATYPE_NULL;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_DOUBLE;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_FLOAT;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_INT;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_SHORT;
+NGCORE_API extern NG_MPI_Datatype NG_MPI_UINT64_T;
+NGCORE_API extern NG_MPI_Op NG_MPI_LOR;
+NGCORE_API extern NG_MPI_Op NG_MPI_MAX;
+NGCORE_API extern NG_MPI_Op NG_MPI_MIN;
+NGCORE_API extern NG_MPI_Op NG_MPI_SUM;
+NGCORE_API extern NG_MPI_Request NG_MPI_REQUEST_NULL;
+NGCORE_API extern NG_MPI_Status* NG_MPI_STATUSES_IGNORE;
+NGCORE_API extern NG_MPI_Status* NG_MPI_STATUS_IGNORE;
+NGCORE_API extern int NG_MPI_ANY_SOURCE;
+NGCORE_API extern int NG_MPI_ANY_TAG;
+NGCORE_API extern int NG_MPI_MAX_PROCESSOR_NAME;
+NGCORE_API extern int NG_MPI_PROC_NULL;
+NGCORE_API extern int NG_MPI_ROOT;
+NGCORE_API extern int NG_MPI_SUBVERSION;
+NGCORE_API extern int NG_MPI_THREAD_MULTIPLE;
+NGCORE_API extern int NG_MPI_THREAD_SINGLE;
+NGCORE_API extern int NG_MPI_VERSION;
+NGCORE_API extern void* NG_MPI_IN_PLACE;
+#else  // NG_MPI_WRAPPER
+#define NG_MPI_Wtime MPI_Wtime
+#define NG_MPI_Allgather MPI_Allgather
+#define NG_MPI_Allreduce MPI_Allreduce
+#define NG_MPI_Alltoall MPI_Alltoall
+#define NG_MPI_Barrier MPI_Barrier
+#define NG_MPI_Bcast MPI_Bcast
+#define NG_MPI_Ibcast MPI_Ibcast
+#define NG_MPI_Comm_c2f MPI_Comm_c2f
+#define NG_MPI_Comm_create MPI_Comm_create
+#define NG_MPI_Comm_create_group MPI_Comm_create_group
+#define NG_MPI_Comm_free MPI_Comm_free
+#define NG_MPI_Comm_group MPI_Comm_group
+#define NG_MPI_Comm_rank MPI_Comm_rank
+#define NG_MPI_Comm_size MPI_Comm_size
+#define NG_MPI_Finalize MPI_Finalize
+#define NG_MPI_Gather MPI_Gather
+#define NG_MPI_Gatherv MPI_Gatherv
+#define NG_MPI_Get_count MPI_Get_count
+#define NG_MPI_Get_processor_name MPI_Get_processor_name
+#define NG_MPI_Group_incl MPI_Group_incl
+#define NG_MPI_Init MPI_Init
+#define NG_MPI_Init_thread MPI_Init_thread
+#define NG_MPI_Initialized MPI_Initialized
+#define NG_MPI_Iprobe MPI_Iprobe
+#define NG_MPI_Irecv MPI_Irecv
+#define NG_MPI_Isend MPI_Isend
+#define NG_MPI_Probe MPI_Probe
+#define NG_MPI_Query_thread MPI_Query_thread
+#define NG_MPI_Recv MPI_Recv
+#define NG_MPI_Recv_init MPI_Recv_init
+#define NG_MPI_Reduce MPI_Reduce
+#define NG_MPI_Reduce_local MPI_Reduce_local
+#define NG_MPI_Request_free MPI_Request_free
+#define NG_MPI_Scatter MPI_Scatter
+#define NG_MPI_Send MPI_Send
+#define NG_MPI_Send_init MPI_Send_init
+#define NG_MPI_Startall MPI_Startall
+#define NG_MPI_Type_commit MPI_Type_commit
+#define NG_MPI_Type_contiguous MPI_Type_contiguous
+#define NG_MPI_Type_create_resized MPI_Type_create_resized
+#define NG_MPI_Type_create_struct MPI_Type_create_struct
+#define NG_MPI_Type_free MPI_Type_free
+#define NG_MPI_Type_get_extent MPI_Type_get_extent
+#define NG_MPI_Type_indexed MPI_Type_indexed
+#define NG_MPI_Type_size MPI_Type_size
+#define NG_MPI_Wait MPI_Wait
+#define NG_MPI_Waitall MPI_Waitall
+#define NG_MPI_Waitany MPI_Waitany
+#define NG_MPI_COMM_NULL MPI_COMM_NULL
+#define NG_MPI_COMM_WORLD MPI_COMM_WORLD
+#define NG_MPI_CHAR MPI_CHAR
+#define NG_MPI_CXX_DOUBLE_COMPLEX MPI_CXX_DOUBLE_COMPLEX
+#define NG_MPI_C_BOOL MPI_C_BOOL
+#define NG_MPI_DATATYPE_NULL MPI_DATATYPE_NULL
+#define NG_MPI_DOUBLE MPI_DOUBLE
+#define NG_MPI_FLOAT MPI_FLOAT
+#define NG_MPI_INT MPI_INT
+#define NG_MPI_SHORT MPI_SHORT
+#define NG_MPI_UINT64_T MPI_UINT64_T
+#define NG_MPI_LOR MPI_LOR
+#define NG_MPI_MAX MPI_MAX
+#define NG_MPI_MIN MPI_MIN
+#define NG_MPI_SUM MPI_SUM
+#define NG_MPI_REQUEST_NULL MPI_REQUEST_NULL
+#define NG_MPI_STATUSES_IGNORE MPI_STATUSES_IGNORE
+#define NG_MPI_STATUS_IGNORE MPI_STATUS_IGNORE
+#define NG_MPI_ANY_SOURCE MPI_ANY_SOURCE
+#define NG_MPI_ANY_TAG MPI_ANY_TAG
+#define NG_MPI_MAX_PROCESSOR_NAME MPI_MAX_PROCESSOR_NAME
+#define NG_MPI_PROC_NULL MPI_PROC_NULL
+#define NG_MPI_ROOT MPI_ROOT
+#define NG_MPI_SUBVERSION MPI_SUBVERSION
+#define NG_MPI_THREAD_MULTIPLE MPI_THREAD_MULTIPLE
+#define NG_MPI_THREAD_SINGLE MPI_THREAD_SINGLE
+#define NG_MPI_VERSION MPI_VERSION
+#define NG_MPI_IN_PLACE MPI_IN_PLACE
+#endif // NG_MPI_WRAPPER

libsrc/core/ng_mpi_generated_dummy_init.hpp

@@ -0,0 +1,76 @@
+decltype(NG_MPI_Wtime) NG_MPI_Wtime = []()->double { throw no_mpi(); };
+decltype(NG_MPI_Allgather) NG_MPI_Allgather = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Allreduce) NG_MPI_Allreduce = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Alltoall) NG_MPI_Alltoall = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Barrier) NG_MPI_Barrier = [](NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Bcast) NG_MPI_Bcast = [](void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Ibcast) NG_MPI_Ibcast = [](void*, int, NG_MPI_Datatype, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_c2f) NG_MPI_Comm_c2f = [](NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_create) NG_MPI_Comm_create = [](NG_MPI_Comm, NG_MPI_Group, NG_MPI_Comm*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_create_group) NG_MPI_Comm_create_group = [](NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_free) NG_MPI_Comm_free = [](NG_MPI_Comm*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_group) NG_MPI_Comm_group = [](NG_MPI_Comm, NG_MPI_Group*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_rank) NG_MPI_Comm_rank = [](NG_MPI_Comm, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Comm_size) NG_MPI_Comm_size = [](NG_MPI_Comm, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Finalize) NG_MPI_Finalize = []()->int { throw no_mpi(); };
+decltype(NG_MPI_Gather) NG_MPI_Gather = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Gatherv) NG_MPI_Gatherv = [](void*, int, NG_MPI_Datatype, void*, int*, int*, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Get_count) NG_MPI_Get_count = [](NG_MPI_Status*, NG_MPI_Datatype, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Get_processor_name) NG_MPI_Get_processor_name = [](char*, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Group_incl) NG_MPI_Group_incl = [](NG_MPI_Group, int, int*, NG_MPI_Group*)->int { throw no_mpi(); };
+decltype(NG_MPI_Init) NG_MPI_Init = [](int*, char***)->int { throw no_mpi(); };
+decltype(NG_MPI_Init_thread) NG_MPI_Init_thread = [](int*, char***, int, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Initialized) NG_MPI_Initialized = [](int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Iprobe) NG_MPI_Iprobe = [](int, int, NG_MPI_Comm, int*, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Irecv) NG_MPI_Irecv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Isend) NG_MPI_Isend = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Probe) NG_MPI_Probe = [](int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Query_thread) NG_MPI_Query_thread = [](int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Recv) NG_MPI_Recv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Recv_init) NG_MPI_Recv_init = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Reduce) NG_MPI_Reduce = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Reduce_local) NG_MPI_Reduce_local = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op)->int { throw no_mpi(); };
+decltype(NG_MPI_Request_free) NG_MPI_Request_free = [](NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Scatter) NG_MPI_Scatter = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Send) NG_MPI_Send = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm)->int { throw no_mpi(); };
+decltype(NG_MPI_Send_init) NG_MPI_Send_init = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Startall) NG_MPI_Startall = [](int, NG_MPI_Request*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_commit) NG_MPI_Type_commit = [](NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_contiguous) NG_MPI_Type_contiguous = [](int, NG_MPI_Datatype, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_create_resized) NG_MPI_Type_create_resized = [](NG_MPI_Datatype, NG_MPI_Aint, NG_MPI_Aint, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_create_struct) NG_MPI_Type_create_struct = [](int, int*, NG_MPI_Aint*, NG_MPI_Datatype*, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_free) NG_MPI_Type_free = [](NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_get_extent) NG_MPI_Type_get_extent = [](NG_MPI_Datatype, NG_MPI_Aint*, NG_MPI_Aint*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_indexed) NG_MPI_Type_indexed = [](int, int*, int*, NG_MPI_Datatype, NG_MPI_Datatype*)->int { throw no_mpi(); };
+decltype(NG_MPI_Type_size) NG_MPI_Type_size = [](NG_MPI_Datatype, int*)->int { throw no_mpi(); };
+decltype(NG_MPI_Wait) NG_MPI_Wait = [](NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Waitall) NG_MPI_Waitall = [](int, NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); };
+decltype(NG_MPI_Waitany) NG_MPI_Waitany = [](int, NG_MPI_Request*, int*, NG_MPI_Status*)->int { throw no_mpi(); };
+NG_MPI_Comm NG_MPI_COMM_NULL = 0;
+NG_MPI_Comm NG_MPI_COMM_WORLD = 0;
+NG_MPI_Datatype NG_MPI_CHAR = 0;
+NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX = 0;
+NG_MPI_Datatype NG_MPI_C_BOOL = 0;
+NG_MPI_Datatype NG_MPI_DATATYPE_NULL = 0;
+NG_MPI_Datatype NG_MPI_DOUBLE = 0;
+NG_MPI_Datatype NG_MPI_FLOAT = 0;
+NG_MPI_Datatype NG_MPI_INT = 0;
+NG_MPI_Datatype NG_MPI_SHORT = 0;
+NG_MPI_Datatype NG_MPI_UINT64_T = 0;
+NG_MPI_Op NG_MPI_LOR = 0;
+NG_MPI_Op NG_MPI_MAX = 0;
+NG_MPI_Op NG_MPI_MIN = 0;
+NG_MPI_Op NG_MPI_SUM = 0;
+NG_MPI_Request NG_MPI_REQUEST_NULL = 0;
+NG_MPI_Status* NG_MPI_STATUSES_IGNORE = 0;
+NG_MPI_Status* NG_MPI_STATUS_IGNORE = 0;
+int NG_MPI_ANY_SOURCE = 0;
+int NG_MPI_ANY_TAG = 0;
+int NG_MPI_MAX_PROCESSOR_NAME = 0;
+int NG_MPI_PROC_NULL = 0;
+int NG_MPI_ROOT = 0;
+int NG_MPI_SUBVERSION = 0;
+int NG_MPI_THREAD_MULTIPLE = 0;
+int NG_MPI_THREAD_SINGLE = 0;
+int NG_MPI_VERSION = 0;
+void* NG_MPI_IN_PLACE = 0;

libsrc/core/ng_mpi_generated_init.hpp

@@ -0,0 +1,76 @@
+NG_MPI_Wtime = []()->double { return MPI_Wtime(); };
+NG_MPI_Allgather = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, NG_MPI_Comm arg6)->int { return MPI_Allgather( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Allreduce = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4, NG_MPI_Comm arg5)->int { return MPI_Allreduce( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4),  ng2mpi(arg5)); };
+NG_MPI_Alltoall = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, NG_MPI_Comm arg6)->int { return MPI_Alltoall( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Barrier = [](NG_MPI_Comm arg0)->int { return MPI_Barrier( ng2mpi(arg0)); };
+NG_MPI_Bcast = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, NG_MPI_Comm arg4)->int { return MPI_Bcast( arg0,  arg1,  ng2mpi(arg2),  arg3,  ng2mpi(arg4)); };
+NG_MPI_Ibcast = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, NG_MPI_Comm arg4, NG_MPI_Request* arg5)->int { return MPI_Ibcast( arg0,  arg1,  ng2mpi(arg2),  arg3,  ng2mpi(arg4),  ng2mpi(arg5)); };
+NG_MPI_Comm_c2f = [](NG_MPI_Comm arg0)->int { return MPI_Comm_c2f( ng2mpi(arg0)); };
+NG_MPI_Comm_create = [](NG_MPI_Comm arg0, NG_MPI_Group arg1, NG_MPI_Comm* arg2)->int { return MPI_Comm_create( ng2mpi(arg0),  ng2mpi(arg1),  ng2mpi(arg2)); };
+NG_MPI_Comm_create_group = [](NG_MPI_Comm arg0, NG_MPI_Group arg1, int arg2, NG_MPI_Comm* arg3)->int { return MPI_Comm_create_group( ng2mpi(arg0),  ng2mpi(arg1),  arg2,  ng2mpi(arg3)); };
+NG_MPI_Comm_free = [](NG_MPI_Comm* arg0)->int { return MPI_Comm_free( ng2mpi(arg0)); };
+NG_MPI_Comm_group = [](NG_MPI_Comm arg0, NG_MPI_Group* arg1)->int { return MPI_Comm_group( ng2mpi(arg0),  ng2mpi(arg1)); };
+NG_MPI_Comm_rank = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_rank( ng2mpi(arg0),  arg1); };
+NG_MPI_Comm_size = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_size( ng2mpi(arg0),  arg1); };
+NG_MPI_Finalize = []()->int { return MPI_Finalize(); };
+NG_MPI_Gather = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, int arg6, NG_MPI_Comm arg7)->int { return MPI_Gather( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  arg6,  ng2mpi(arg7)); };
+NG_MPI_Gatherv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int* arg4, int* arg5, NG_MPI_Datatype arg6, int arg7, NG_MPI_Comm arg8)->int { return MPI_Gatherv( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  arg5,  ng2mpi(arg6),  arg7,  ng2mpi(arg8)); };
+NG_MPI_Get_count = [](NG_MPI_Status* arg0, NG_MPI_Datatype arg1, int* arg2)->int { return MPI_Get_count( ng2mpi(arg0),  ng2mpi(arg1),  arg2); };
+NG_MPI_Get_processor_name = [](char* arg0, int* arg1)->int { return MPI_Get_processor_name( arg0,  arg1); };
+NG_MPI_Group_incl = [](NG_MPI_Group arg0, int arg1, int* arg2, NG_MPI_Group* arg3)->int { return MPI_Group_incl( ng2mpi(arg0),  arg1,  arg2,  ng2mpi(arg3)); };
+NG_MPI_Init = [](int* arg0, char*** arg1)->int { return MPI_Init( arg0,  arg1); };
+NG_MPI_Init_thread = [](int* arg0, char*** arg1, int arg2, int* arg3)->int { return MPI_Init_thread( arg0,  arg1,  arg2,  arg3); };
+NG_MPI_Initialized = [](int* arg0)->int { return MPI_Initialized( arg0); };
+NG_MPI_Iprobe = [](int arg0, int arg1, NG_MPI_Comm arg2, int* arg3, NG_MPI_Status* arg4)->int { return MPI_Iprobe( arg0,  arg1,  ng2mpi(arg2),  arg3,  ng2mpi(arg4)); };
+NG_MPI_Irecv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Irecv( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Isend = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Isend( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Probe = [](int arg0, int arg1, NG_MPI_Comm arg2, NG_MPI_Status* arg3)->int { return MPI_Probe( arg0,  arg1,  ng2mpi(arg2),  ng2mpi(arg3)); };
+NG_MPI_Query_thread = [](int* arg0)->int { return MPI_Query_thread( arg0); };
+NG_MPI_Recv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Status* arg6)->int { return MPI_Recv( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Recv_init = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Recv_init( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Reduce = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4, int arg5, NG_MPI_Comm arg6)->int { return MPI_Reduce( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4),  arg5,  ng2mpi(arg6)); };
+NG_MPI_Reduce_local = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4)->int { return MPI_Reduce_local( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4)); };
+NG_MPI_Request_free = [](NG_MPI_Request* arg0)->int { return MPI_Request_free( ng2mpi(arg0)); };
+NG_MPI_Scatter = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, int arg6, NG_MPI_Comm arg7)->int { return MPI_Scatter( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  arg6,  ng2mpi(arg7)); };
+NG_MPI_Send = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5)->int { return MPI_Send( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5)); };
+NG_MPI_Send_init = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Send_init( arg0,  arg1,  ng2mpi(arg2),  arg3,  arg4,  ng2mpi(arg5),  ng2mpi(arg6)); };
+NG_MPI_Startall = [](int arg0, NG_MPI_Request* arg1)->int { return MPI_Startall( arg0,  ng2mpi(arg1, arg0)); };
+NG_MPI_Type_commit = [](NG_MPI_Datatype* arg0)->int { return MPI_Type_commit( ng2mpi(arg0)); };
+NG_MPI_Type_contiguous = [](int arg0, NG_MPI_Datatype arg1, NG_MPI_Datatype* arg2)->int { return MPI_Type_contiguous( arg0,  ng2mpi(arg1),  ng2mpi(arg2)); };
+NG_MPI_Type_create_resized = [](NG_MPI_Datatype arg0, NG_MPI_Aint arg1, NG_MPI_Aint arg2, NG_MPI_Datatype* arg3)->int { return MPI_Type_create_resized( ng2mpi(arg0),  ng2mpi(arg1),  ng2mpi(arg2),  ng2mpi(arg3)); };
+NG_MPI_Type_create_struct = [](int arg0, int* arg1, NG_MPI_Aint* arg2, NG_MPI_Datatype* arg3, NG_MPI_Datatype* arg4)->int { return MPI_Type_create_struct( arg0,  arg1,  ng2mpi(arg2, arg0),  ng2mpi(arg3, arg0),  ng2mpi(arg4)); };
+NG_MPI_Type_free = [](NG_MPI_Datatype* arg0)->int { return MPI_Type_free( ng2mpi(arg0)); };
+NG_MPI_Type_get_extent = [](NG_MPI_Datatype arg0, NG_MPI_Aint* arg1, NG_MPI_Aint* arg2)->int { return MPI_Type_get_extent( ng2mpi(arg0),  ng2mpi(arg1),  ng2mpi(arg2)); };
+NG_MPI_Type_indexed = [](int arg0, int* arg1, int* arg2, NG_MPI_Datatype arg3, NG_MPI_Datatype* arg4)->int { return MPI_Type_indexed( arg0,  arg1,  arg2,  ng2mpi(arg3),  ng2mpi(arg4)); };
+NG_MPI_Type_size = [](NG_MPI_Datatype arg0, int* arg1)->int { return MPI_Type_size( ng2mpi(arg0),  arg1); };
+NG_MPI_Wait = [](NG_MPI_Request* arg0, NG_MPI_Status* arg1)->int { return MPI_Wait( ng2mpi(arg0),  ng2mpi(arg1)); };
+NG_MPI_Waitall = [](int arg0, NG_MPI_Request* arg1, NG_MPI_Status* arg2)->int { return MPI_Waitall( arg0,  ng2mpi(arg1, arg0),  ng2mpi(arg2)); };
+NG_MPI_Waitany = [](int arg0, NG_MPI_Request* arg1, int* arg2, NG_MPI_Status* arg3)->int { return MPI_Waitany( arg0,  ng2mpi(arg1, arg0),  arg2,  ng2mpi(arg3)); };
+NG_MPI_COMM_NULL = mpi2ng(MPI_COMM_NULL);
+NG_MPI_COMM_WORLD = mpi2ng(MPI_COMM_WORLD);
+NG_MPI_CHAR = mpi2ng(MPI_CHAR);
+NG_MPI_CXX_DOUBLE_COMPLEX = mpi2ng(MPI_CXX_DOUBLE_COMPLEX);
+NG_MPI_C_BOOL = mpi2ng(MPI_C_BOOL);
+NG_MPI_DATATYPE_NULL = mpi2ng(MPI_DATATYPE_NULL);
+NG_MPI_DOUBLE = mpi2ng(MPI_DOUBLE);
+NG_MPI_FLOAT = mpi2ng(MPI_FLOAT);
+NG_MPI_INT = mpi2ng(MPI_INT);
+NG_MPI_SHORT = mpi2ng(MPI_SHORT);
+NG_MPI_UINT64_T = mpi2ng(MPI_UINT64_T);
+NG_MPI_LOR = mpi2ng(MPI_LOR);
+NG_MPI_MAX = mpi2ng(MPI_MAX);
+NG_MPI_MIN = mpi2ng(MPI_MIN);
+NG_MPI_SUM = mpi2ng(MPI_SUM);
+NG_MPI_REQUEST_NULL = mpi2ng(MPI_REQUEST_NULL);
+NG_MPI_STATUSES_IGNORE = mpi2ng(MPI_STATUSES_IGNORE);
+NG_MPI_STATUS_IGNORE = mpi2ng(MPI_STATUS_IGNORE);
+NG_MPI_ANY_SOURCE = mpi2ng(MPI_ANY_SOURCE);
+NG_MPI_ANY_TAG = mpi2ng(MPI_ANY_TAG);
+NG_MPI_MAX_PROCESSOR_NAME = mpi2ng(MPI_MAX_PROCESSOR_NAME);
+NG_MPI_PROC_NULL = mpi2ng(MPI_PROC_NULL);
+NG_MPI_ROOT = mpi2ng(MPI_ROOT);
+NG_MPI_SUBVERSION = mpi2ng(MPI_SUBVERSION);
+NG_MPI_THREAD_MULTIPLE = mpi2ng(MPI_THREAD_MULTIPLE);
+NG_MPI_THREAD_SINGLE = mpi2ng(MPI_THREAD_SINGLE);
+NG_MPI_VERSION = mpi2ng(MPI_VERSION);
+NG_MPI_IN_PLACE = mpi2ng(MPI_IN_PLACE);

libsrc/core/ng_mpi_native.hpp

@@ -0,0 +1,21 @@
+#ifndef NG_MPI_NATIVE_HPP
+#define NG_MPI_NATIVE_HPP
+
+#include <mpi.h>
+
+#include "mpi_wrapper.hpp"
+#include "ng_mpi.hpp"
+
+namespace ngcore {
+
+MPI_Comm NG_MPI_Native(NG_MPI_Comm comm) {
+  return reinterpret_cast<MPI_Comm>(comm.value);
+}
+
+MPI_Comm NG_MPI_Native(NgMPI_Comm comm) {
+  return reinterpret_cast<MPI_Comm>(static_cast<NG_MPI_Comm>(comm).value);
+}
+
+}  // namespace ngcore
+
+#endif  // NG_MPI_NATIVE_HPP

libsrc/core/ng_mpi_wrapper.cpp

@@ -0,0 +1,206 @@
+#ifdef PARALLEL
+
+#include <filesystem>
+#include <iostream>
+#include <stdexcept>
+
+#include "ng_mpi.hpp"
+#include "ngstream.hpp"
+#ifdef NG_PYTHON
+#include "python_ngcore.hpp"
+#endif // NG_PYTHON
+#include "utils.hpp"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+
+#ifndef NG_MPI_WRAPPER
+#ifdef NG_PYTHON
+#define MPI4PY_LIMITED_API 1
+#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1
+#define MPI4PY_LIMITED_API_SKIP_SESSION 1
+#include "mpi4py_pycapi.h"  // mpi4py < 4.0.0
+#endif // NG_PYTHON
+#endif // NG_MPI_WRAPPER
+
+namespace ngcore {
+
+#ifdef NG_MPI_WRAPPER
+static std::unique_ptr<SharedLibrary> mpi_lib, ng_mpi_lib;
+static bool need_mpi_finalize = false;
+
+struct MPIFinalizer {
+  ~MPIFinalizer() {
+    if (need_mpi_finalize) {
+      cout << IM(5) << "Calling MPI_Finalize" << endl;
+      NG_MPI_Finalize();
+    }
+  }
+} mpi_finalizer;
+
+bool MPI_Loaded() { return ng_mpi_lib != nullptr; }
+
+void InitMPI(std::optional<std::filesystem::path> mpi_lib_path) {
+  if (ng_mpi_lib) return;
+
+  cout << IM(3) << "InitMPI" << endl;
+
+  std::string vendor = "";
+  std::string mpi4py_lib_file = "";
+
+  if (mpi_lib_path) {
+    // Dynamic load of given shared MPI library
+    // Then call MPI_Init, read the library version and set the vender name
+    try {
+      typedef int (*init_handle)(int *, char ***);
+      typedef int (*mpi_initialized_handle)(int *);
+      mpi_lib =
+          std::make_unique<SharedLibrary>(*mpi_lib_path, std::nullopt, true);
+      auto mpi_init = mpi_lib->GetSymbol<init_handle>("MPI_Init");
+      auto mpi_initialized =
+          mpi_lib->GetSymbol<mpi_initialized_handle>("MPI_Initialized");
+
+      int flag = 0;
+      mpi_initialized(&flag);
+      if (!flag) {
+        typedef const char *pchar;
+        int argc = 1;
+        pchar args[] = {"netgen", nullptr};
+        pchar *argv = &args[0];
+        cout << IM(5) << "Calling MPI_Init" << endl;
+        mpi_init(&argc, (char ***)argv);
+        need_mpi_finalize = true;
+      }
+
+      char c_version_string[65536];
+      c_version_string[0] = '\0';
+      int result_len = 0;
+      typedef void (*get_version_handle)(char *, int *);
+      auto get_version =
+          mpi_lib->GetSymbol<get_version_handle>("MPI_Get_library_version");
+      get_version(c_version_string, &result_len);
+      std::string version = c_version_string;
+
+      if (version.substr(0, 8) == "Open MPI")
+        vendor = "Open MPI";
+      else if (version.substr(0, 5) == "MPICH")
+        vendor = "MPICH";
+      else if (version.substr(0, 13) == "Microsoft MPI")
+        vendor = "Microsoft MPI";
+      else if (version.substr(0, 12) == "Intel(R) MPI")
+        vendor = "Intel MPI";
+      else
+        throw std::runtime_error(
+            std::string("Unknown MPI version: " + version));
+    } catch (std::runtime_error &e) {
+      cerr << "Could not load MPI: " << e.what() << endl;
+      throw e;
+    }
+  } else {
+#ifdef NG_PYTHON
+    // Use mpi4py to init MPI library and get the vendor name
+    auto mpi4py = py::module::import("mpi4py.MPI");
+    vendor = mpi4py.attr("get_vendor")()[py::int_(0)].cast<std::string>();
+
+#ifndef WIN32
+    // Load mpi4py library (it exports all MPI symbols) to have all MPI symbols
+    // available before the ng_mpi wrapper is loaded This is not necessary on
+    // windows as the matching mpi dll is linked to the ng_mpi wrapper directly
+    mpi4py_lib_file = mpi4py.attr("__file__").cast<std::string>();
+    mpi_lib =
+        std::make_unique<SharedLibrary>(mpi4py_lib_file, std::nullopt, true);
+#endif  // WIN32
+#endif // NG_PYTHON
+  }
+
+  std::string ng_lib_name = "";
+  if (vendor == "Open MPI")
+    ng_lib_name = "ng_openmpi";
+  else if (vendor == "MPICH")
+    ng_lib_name = "ng_mpich";
+  else if (vendor == "Microsoft MPI")
+    ng_lib_name = "ng_microsoft_mpi";
+  else if (vendor == "Intel MPI")
+    ng_lib_name = "ng_intel_mpi";
+  else
+    throw std::runtime_error("Unknown MPI vendor: " + vendor);
+
+  ng_lib_name += NETGEN_SHARED_LIBRARY_SUFFIX;
+
+  // Load the ng_mpi wrapper and call ng_init_mpi to set all function pointers
+  typedef void (*ng_init_handle)();
+  ng_mpi_lib = std::make_unique<SharedLibrary>(ng_lib_name);
+  ng_mpi_lib->GetSymbol<ng_init_handle>("ng_init_mpi")();
+  std::cout << IM(3) << "MPI wrapper loaded, vendor: " << vendor << endl;
+}
+
+static std::runtime_error no_mpi() {
+  return std::runtime_error("MPI not enabled");
+}
+
+#ifdef NG_PYTHON
+decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
+    [](py::handle py_obj, NG_MPI_Comm &ng_comm) -> bool {
+  // If this gets called, it means that we want to convert an mpi4py
+  // communicator to a Netgen MPI communicator, but the Netgen MPI wrapper
+  // runtime was not yet initialized.
+
+  // store the current address of this function
+  auto old_converter_address = NG_MPI_CommFromMPI4Py;
+
+  // initialize the MPI wrapper runtime, this sets all the function pointers
+  InitMPI();
+
+  // if the initialization was successful, the function pointer should have
+  // changed
+  // -> call the actual conversion function
+  if (NG_MPI_CommFromMPI4Py != old_converter_address)
+    return NG_MPI_CommFromMPI4Py(py_obj, ng_comm);
+
+  // otherwise, something strange happened
+  throw no_mpi();
+};
+decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
+    [](NG_MPI_Comm) -> py::handle { throw no_mpi(); };
+#endif  // NG_PYTHON
+
+#include "ng_mpi_generated_dummy_init.hpp"
+#else  // NG_MPI_WRAPPER
+
+static bool imported_mpi4py = false;
+#ifdef NG_PYTHON
+decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
+    [](py::handle src, NG_MPI_Comm &dst) -> bool {
+  if (!imported_mpi4py) {
+    import_mpi4py__MPI();
+    imported_mpi4py = true;
+  }
+  PyObject *py_src = src.ptr();
+  auto type = Py_TYPE(py_src);
+  if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
+    dst = *PyMPIComm_Get(py_src);
+    return !PyErr_Occurred();
+  }
+  return false;
+};
+
+decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
+    [](NG_MPI_Comm src) -> py::handle {
+  if (!imported_mpi4py) {
+    import_mpi4py__MPI();
+    imported_mpi4py = true;
+  }
+  return py::handle(PyMPIComm_New(src));
+};
+
+#endif  // NG_PYTHON
+
+bool MPI_Loaded() { return true; }
+void InitMPI(std::optional<std::filesystem::path>) {}
+
+#endif  // NG_MPI_WRAPPER
+
+}  // namespace ngcore
+
+#endif  // PARALLEL

libsrc/core/ngcore.hpp

@@ -6,16 +6,22 @@
 #include "bitarray.hpp"
 #include "exception.hpp"
 #include "flags.hpp"
+#include "table.hpp"
 #include "hashtable.hpp"
 #include "localheap.hpp"
 #include "logging.hpp"
-#include "mpi_wrapper.hpp"
+// #include "mpi_wrapper.hpp"
 #include "profiler.hpp"
 #include "signal.hpp"
+#include "simd.hpp"
+#include "autodiff.hpp"
+#include "autodiffdiff.hpp"
 #include "symboltable.hpp"
-#include "table.hpp"
 #include "taskmanager.hpp"
 #include "version.hpp"
 #include "xbool.hpp"
+#include "ngstream.hpp"
+#include "utils.hpp"
+#include "ranges.hpp"
 
 #endif // NETGEN_CORE_NGCORE_HPP

libsrc/core/ngcore_api.hpp

@@ -1,6 +1,8 @@
 #ifndef NETGEN_CORE_NGCORE_API_HPP
 #define NETGEN_CORE_NGCORE_API_HPP
 
+#include "netgen_config.hpp"
+
 #ifdef WIN32
 
 // This function or variable may be unsafe. Consider using _ftime64_s instead. To disable deprecation, use _CRT_SECURE_NO_WARNINGS. See online help for details.
@@ -48,25 +50,46 @@
         #define NGCORE_API NGCORE_API_IMPORT
 #endif
 
+// Set __host__ __device__ for all inline functions
+#ifdef __CUDACC__
+  #define NETGEN_HD __host__ __device__
+#else // __CUDACC__
+  #define NETGEN_HD
+#endif // __CUDACC__
+
 #ifdef __INTEL_COMPILER
+  #define NETGEN_ALWAYS_INLINE __forceinline
+  #define NETGEN_INLINE __forceinline inline
   #ifdef WIN32
-    #define NETGEN_INLINE __forceinline inline
     #define NETGEN_LAMBDA_INLINE
   #else
-    #define NETGEN_INLINE __forceinline inline
     #define NETGEN_LAMBDA_INLINE __attribute__ ((__always_inline__))
   #endif
 #else
   #ifdef __GNUC__
-    #define NETGEN_INLINE __attribute__ ((__always_inline__)) inline
-    #define NETGEN_LAMBDA_INLINE __attribute__ ((__always_inline__))
+    #define NETGEN_ALWAYS_INLINE __attribute__ ((__always_inline__))
+    #define NETGEN_INLINE __attribute__ ((__always_inline__)) inline NETGEN_HD
+    #define NETGEN_LAMBDA_INLINE __attribute__ ((__always_inline__)) NETGEN_HD
     #define NETGEN_VLA
   #else
+    #define NETGEN_ALWAYS_INLINE
     #define NETGEN_INLINE inline
     #define NETGEN_LAMBDA_INLINE
   #endif
 #endif
 
+#if defined(__amd64__) || defined(_M_AMD64)
+#define NETGEN_ARCH_AMD64
+#endif
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+#define NETGEN_ARCH_ARM64
+#endif
+
+#if defined(__arm__) || defined(_M_ARM)
+#define NETGEN_ARCH_ARM
+#endif
+
 #ifdef __MAC_OS_X_VERSION_MIN_REQUIRED
 #if __MAC_OS_X_VERSION_MIN_REQUIRED < 101400
 // The c++ standard library on MacOS 10.13 and earlier has no aligned new operator,

libsrc/core/ngstream.hpp

@@ -0,0 +1,115 @@
+#ifndef FILE_NGSTREAM
+#define FILE_NGSTREAM
+
+/**************************************************************************/
+/* File:   ng(s)stream.hpp                                                */
+/* Author: Joachim Schoeberl                                              */
+/* Date:   20. Jul. 2011                                                  */
+/**************************************************************************/
+
+// #include <ios>
+// #include <iostream>
+namespace ngcore
+{
+
+  NGCORE_API extern int printmessage_importance;
+ 
+  // important message
+  class IM
+  {
+    int value;
+  public:
+    IM (int val) : value(val) { ; }
+    int Value () const { return value; }
+  };
+
+  class trunc
+  {
+    double eps;
+  public:
+    trunc (double aeps) : eps(aeps) { ; }
+    double Eps() const { return eps; }
+  };
+  
+  class NGSOStream
+  {
+    std::ostream & ost;
+    bool active;
+    NGCORE_API static bool glob_active;
+    double trunc;
+  public:
+    NGSOStream (std::ostream & aost, bool aactive)
+      : ost(aost), active(aactive), trunc(-1) { ; }
+    NGSOStream & SetTrunc (double atrunc) { trunc = atrunc; return *this; }
+    double GetTrunc () const { return trunc; }
+    bool Active () const { return active && glob_active; }
+    std::ostream & GetStream () { return ost; }
+    static void SetGlobalActive (bool b) { glob_active = b; }
+  };
+  
+  inline NGSOStream operator<< (std::ostream & ost, const IM & im)
+  {
+    return NGSOStream (ost, 
+		       (im.Value() <= printmessage_importance));
+  }
+
+  /*
+    // doesn't work for matrices
+  inline NGSOStream operator<< (ostream & ost, trunc tr)
+  {
+    cout << "set trunc modifier" << endl;
+    return NGSOStream (ost, true).SetTrunc (tr.Eps());
+  }
+  */
+
+  
+  template <typename T>
+  inline NGSOStream operator<< (NGSOStream ngsost, const T & data)
+  {
+    if (ngsost.Active())
+      ngsost.GetStream() << data;
+    return ngsost;
+  }
+
+  /*
+  inline NGSOStream operator<< (NGSOStream ngsost, const double & data)
+  {
+    cout << "double out" << endl;
+    if (ngsost.Active())
+      {
+        double hdata = data;
+        if (fabs (hdata) < ngsost.GetTrunc()) hdata = 0.0;
+        ngsost.GetStream() << hdata;
+      }
+    return ngsost;
+  }
+  */
+  
+  inline NGSOStream operator<< (NGSOStream ngsost, std::ostream& ( *pf )(std::ostream&))
+  {
+    if ( ngsost.Active() )
+      ngsost.GetStream() << (*pf);
+
+    return ngsost;
+  }
+  
+  inline NGSOStream operator<< (NGSOStream ngsost, std::ios& ( *pf )(std::ios&))
+  {
+    if ( ngsost.Active() )
+      ngsost.GetStream() << (*pf);
+    
+    return ngsost;
+  }
+
+  inline NGSOStream operator<< (NGSOStream ngsost, std::ios_base& ( *pf )(std::ios_base&))
+  {
+    if ( ngsost.Active() )
+      ngsost.GetStream() << (*pf);
+    
+    return ngsost;
+  }
+
+
+}
+
+#endif

libsrc/core/paje_trace.cpp

@@ -7,29 +7,50 @@
 
 #include "archive.hpp"           // for Demangle
 #include "paje_trace.hpp"
+#include "ng_mpi.hpp"
 #include "profiler.hpp"
+#include "mpi_wrapper.hpp"
 
 extern const char *header;
 
+constexpr int MPI_PAJE_WRITER = 1;
+constexpr int ASSUMED_MPI_MAX_PROCESSOR_NAME = 256;
+
 namespace ngcore
 {
+  static std::string GetTimerName( int id )
+  {
+#ifndef PARALLEL
+    return NgProfiler::GetName(id);
+#else // PARALLEL
+    if(id<NgProfiler::SIZE)
+      return NgProfiler::GetName(id);
+
+    NgMPI_Comm comm(NG_MPI_COMM_WORLD);
+    return NgProfiler::GetName(id-NgProfiler::SIZE*comm.Rank());
+#endif // PARALLEL
+  }
+
+  std::vector<PajeTrace::MemoryEvent> PajeTrace::memory_events;
+
   // Produce no traces by default
   size_t PajeTrace::max_tracefile_size = 0;
 
   // If true, produce variable counting active threads
   // increases trace by a factor of two
-  bool PajeTrace::trace_thread_counter = true;
+  bool PajeTrace::trace_thread_counter = false;
   bool PajeTrace::trace_threads = true;
+  bool PajeTrace::mem_tracing_enabled = true;
+  bool PajeTrace::write_paje_file = true;
 
   PajeTrace :: PajeTrace(int anthreads, std::string aname)
   {
-    start_time = GetTimeCounter();
 
     nthreads = anthreads;
     tracefile_name = std::move(aname);
 
     int bytes_per_event=33;
-    max_num_events_per_thread = std::min( static_cast<size_t>(std::numeric_limits<int>::max()), max_tracefile_size/bytes_per_event/(2*nthreads+1)*10/7);
+    max_num_events_per_thread = std::min( static_cast<size_t>(std::numeric_limits<int>::max()), max_tracefile_size/bytes_per_event/(nthreads+1+trace_thread_counter*nthreads)*10/7);
     if(max_num_events_per_thread>0)
     {
       logger->info( "Tracefile size = {}MB", max_tracefile_size/1024/1024);
@@ -47,14 +68,79 @@ namespace ngcore
 
     jobs.reserve(reserve_size);
     timer_events.reserve(reserve_size);
+    gpu_events.reserve(reserve_size);
+    memory_events.reserve(1024*1024);
 
+    // sync start time when running in parallel
+#ifdef PARALLEL
+    if(MPI_Loaded())
+    {
+      NgMPI_Comm comm(NG_MPI_COMM_WORLD);
+      for([[maybe_unused]] auto i : Range(5))
+          comm.Barrier();
+    }
+#endif // PARALLEL
+
+    start_time = GetTimeCounter();
     tracing_enabled = true;
+    mem_tracing_enabled = true;
+    n_memory_events_at_start = memory_events.size();
   }
 
   PajeTrace :: ~PajeTrace()
   {
-    if(!tracefile_name.empty())
-      Write(tracefile_name);
+    for(auto & ltask : tasks)
+        for(auto & task : ltask)
+          {
+            task.time -= start_time;
+          }
+    for(auto & job : jobs)
+      {
+        job.start_time -= start_time;
+        job.stop_time -= start_time;
+      }
+    for(auto & event : timer_events)
+        event.time -= start_time;
+
+    for(auto & event : user_events)
+      {
+        event.t_start -= start_time;
+        event.t_end -= start_time;
+      }
+
+    for(auto & event : gpu_events)
+        event.time -= start_time;
+
+    for(auto & llink : links)
+        for(auto & link : llink)
+            link.time -= start_time;
+
+    for(auto i : IntRange(n_memory_events_at_start, memory_events.size()))
+      memory_events[i].time -= start_time;
+
+    NgMPI_Comm comm;
+  #ifdef PARALLEL
+    if(MPI_Loaded())
+      comm = NgMPI_Comm(NG_MPI_COMM_WORLD);
+  #endif
+    if(comm.Size()==1)
+    {
+      Write();
+    }
+    else
+    {
+      // make sure the timer id is unique across all ranks
+      for(auto & event : timer_events)
+        event.timer_id += NgProfiler::SIZE*comm.Rank();
+
+      for(auto & event : gpu_events)
+        event.timer_id += NgProfiler::SIZE*comm.Rank();
+
+      if(comm.Rank() == MPI_PAJE_WRITER)
+        Write();
+      else
+        SendData();
+    }
   }
 
 
@@ -84,13 +170,12 @@ namespace ngcore
           else if (x<5*d)
             r=6*(x-4*d), g=0,b=1;
           else
-            r=1, g=0,b=1-5*(x-d);
+            r=1, g=0,b=1-6*(x-5*d);
         };
 
       int alias_counter;
 
       FILE * ctrace_stream;
-      TTimePoint start_time;
       std::shared_ptr<Logger> logger = GetLogger("PajeTrace");
 
 
@@ -98,7 +183,7 @@ namespace ngcore
           // return time in milliseconds as double
         // return std::chrono::duration<double>(t-start_time).count()*1000.0;
         // return std::chrono::duration<double>(t-start_time).count() / 2.7e3;
-        return 1000.0*static_cast<double>(t-start_time) * seconds_per_tick;
+        return 1000.0*static_cast<double>(t) * seconds_per_tick;
       }
 
       enum PType
@@ -122,6 +207,10 @@ namespace ngcore
             : time(atime), event_type(aevent_type), type(atype), container(acontainer), value(avalue), id(aid), value_is_alias(avalue_is_alias)
             { }
 
+          PajeEvent( int aevent_type, double atime, int atype, int acontainer, std::string as_value, int aid = 0 )
+            : time(atime), event_type(aevent_type), type(atype), container(acontainer), s_value(as_value), id(aid), value_is_alias(false), value_is_int(false)
+            { }
+
           PajeEvent( int aevent_type, double atime, int atype, int acontainer, int avalue, int astart_container, int akey )
             : time(atime), event_type(aevent_type), type(atype), container(acontainer), value(avalue), start_container(astart_container), id(akey)
             { }
@@ -131,10 +220,12 @@ namespace ngcore
           int event_type;
           int type;
           int container;
+          std::string s_value = "";
           int value = 0;
           int start_container = 0;
           int id = 0;
           bool value_is_alias = true;
+          bool value_is_int = true;
 
           bool operator < (const PajeEvent & other) const {
               // Same start and stop times can occur for very small tasks -> take "starting" events first (eg. PajePushState before PajePopState)
@@ -158,8 +249,10 @@ namespace ngcore
                 case PajePushState:
                   if(value_is_alias)
                     return fprintf( stream, "%d\t%.15g\ta%d\ta%d\ta%d\t%d\n", PajePushState, time, type, container, value, id); // NOLINT
-                  else
+                  else if(value_is_int)
                     return fprintf( stream, "%d\t%.15g\ta%d\ta%d\t%d\t%d\n", PajePushState, time, type, container, value, id); // NOLINT
+                  else
+                    return fprintf( stream, "%d\t%.15g\ta%d\ta%d\t\"%s\"\t%d\n", PajePushState, time, type, container, s_value.c_str(), id); // NOLINT
                 case PajePopState:
                   return fprintf( stream, "%d\t%.15g\ta%d\ta%d\n", PajePopState, time, type, container ); // NOLINT
                 case PajeStartLink:
@@ -180,10 +273,10 @@ namespace ngcore
       void operator=(const PajeFile &) = delete;
       void operator=(PajeFile &&) = delete;
 
-      PajeFile( const std::string & filename, TTimePoint astart_time )
+      PajeFile( const std::string & filename)
         {
-          start_time = astart_time;
-          ctrace_stream = fopen (filename.c_str(),"w"); // NOLINT
+          std::string fname = filename + ".trace";
+          ctrace_stream = fopen (fname.c_str(),"w"); // NOLINT
           fprintf(ctrace_stream, "%s", header ); // NOLINT
           alias_counter = 0;
         }
@@ -284,6 +377,11 @@ namespace ngcore
           events.emplace_back( PajeEvent( PajePushState, ConvertTime(time), type, container, value, id, value_is_alias) );
         }
 
+      void PushState ( TTimePoint time, int type, int container, std::string value, int id = 0)
+        {
+          events.emplace_back( PajeEvent( PajePushState, ConvertTime(time), type, container, value, id) );
+        }
+
       void PopState ( TTimePoint time, int type, int container )
         {
           events.emplace_back( PajeEvent( PajePopState, ConvertTime(time), type, container ) );
@@ -346,7 +444,16 @@ namespace ngcore
 
   NGCORE_API PajeTrace *trace;
 
-  void PajeTrace::Write( const std::string & filename )
+  void PajeTrace::Write( )
+    {
+      if(write_paje_file) WritePajeFile( tracefile_name );
+      WriteTimingChart();
+#ifdef NETGEN_TRACE_MEMORY
+      WriteMemoryChart("");
+#endif // NETGEN_TRACE_MEMORY
+    }
+
+  void PajeTrace::WritePajeFile( const std::string & filename )
     {
       auto n_events = jobs.size() + timer_events.size();
       for(auto & vtasks : tasks)
@@ -365,39 +472,86 @@ namespace ngcore
             logger->warn("Tracing stopped during computation due to tracefile size limit of {} megabytes.", max_tracefile_size/1024/1024);
         }
 
-      PajeFile paje(filename, start_time);
+      PajeFile paje(filename);
 
       const int container_type_task_manager = paje.DefineContainerType( 0, "Task Manager" );
       const int container_type_node = paje.DefineContainerType( container_type_task_manager, "Node");
       const int container_type_thread = paje.DefineContainerType( container_type_task_manager, "Thread");
       const int container_type_timer = container_type_thread; //paje.DefineContainerType( container_type_task_manager, "Timers");
       const int container_type_jobs = paje.DefineContainerType( container_type_task_manager, "Jobs");
+      const int container_type_memory = paje.DefineContainerType( container_type_task_manager, "Memory usage");
 
       const int state_type_job = paje.DefineStateType( container_type_jobs, "Job" );
       const int state_type_task = paje.DefineStateType( container_type_thread, "Task" );
       const int state_type_timer = paje.DefineStateType( container_type_timer, "Timer state" );
 
-      const int variable_type_active_threads = paje.DefineVariableType( container_type_jobs, "Active threads" );
+      int variable_type_active_threads = 0;
+      if(trace_thread_counter)
+          variable_type_active_threads = paje.DefineVariableType( container_type_jobs, "Active threads" );
 
       const int container_task_manager = paje.CreateContainer( container_type_task_manager, 0, "The task manager" );
       const int container_jobs = paje.CreateContainer( container_type_jobs, container_task_manager, "Jobs" );
-      paje.SetVariable( start_time, variable_type_active_threads, container_jobs, 0.0 );
 
-      const int num_nodes = 1; //task_manager ? task_manager->GetNumNodes() : 1;
+      int variable_type_memory = 0;
+      const int container_memory = paje.CreateContainer( container_type_memory, container_task_manager, "Memory" );
+      if(mem_tracing_enabled)
+      {
+        variable_type_memory = paje.DefineVariableType( container_type_task_manager, "Memory [MB]" );
+      }
 
+
+      int num_nodes = 1; //task_manager ? task_manager->GetNumNodes() : 1;
+      std::vector <int> thread_aliases;
       std::vector<int> container_nodes;
-      container_nodes.reserve(num_nodes);
-      for(int i=0; i<num_nodes; i++)
+
+      NgMPI_Comm comm;
+  #ifdef PARALLEL
+      if(MPI_Loaded())
+        comm = NgMPI_Comm(NG_MPI_COMM_WORLD);
+      if(comm.Size()>1)
+      {
+        auto comm = NgMPI_Comm(NG_MPI_COMM_WORLD);
+        nthreads = comm.Size();
+        thread_aliases.reserve(nthreads);
+
+        std::array<char, ASSUMED_MPI_MAX_PROCESSOR_NAME+1> ahostname;
+        int len;
+        NG_MPI_Get_processor_name(ahostname.data(), &len);
+        std::string hostname = ahostname.data();
+
+        std::map<std::string, int> host_map;
+
+        std::string name;
+        for(auto i : IntRange(0, comm.Size()))
+        {
+          if(i!=MPI_PAJE_WRITER)
+            comm.Recv(name, i, 0);
+          else
+            name = hostname;
+          if(host_map.count(name)==0)
+          {
+            host_map[name] = container_nodes.size();
+            container_nodes.emplace_back( paje.CreateContainer( container_type_node, container_task_manager, name) );
+          }
+          thread_aliases.emplace_back( paje.CreateContainer( container_type_thread, container_nodes[host_map[name]], "Rank " + ToString(i) ) );
+        }
+      }
+      else
+  #endif
+      {
+        container_nodes.reserve(num_nodes);
+        for(int i=0; i<num_nodes; i++)
           container_nodes.emplace_back( paje.CreateContainer( container_type_node, container_task_manager, "Node " + ToString(i)) );
 
-      std::vector <int> thread_aliases;
-      thread_aliases.reserve(nthreads);
-      if(trace_threads)
-        for (int i=0; i<nthreads; i++)
+        thread_aliases.reserve(nthreads);
+        if(trace_threads)
+          for (int i=0; i<nthreads; i++)
           {
             auto name = "Thread " + ToString(i);
-            thread_aliases.emplace_back( paje.CreateContainer( container_type_thread, container_nodes[i*num_nodes/nthreads], name ) );
+            if(tasks[i].size())
+              thread_aliases.emplace_back( paje.CreateContainer( container_type_thread, container_nodes[i*num_nodes/nthreads], name ) );
           }
+      }
 
       std::map<const std::type_info *, int> job_map;
       std::map<const std::type_info *, int> job_task_map;
@@ -416,20 +570,73 @@ namespace ngcore
           paje.PopState( j.stop_time, state_type_job, container_jobs );
         }
 
+      size_t memory_at_start = 0;
+
+      for(const auto & i : IntRange(0, n_memory_events_at_start))
+      {
+        if(memory_events[i].is_alloc)
+            memory_at_start += memory_events[i].size;
+        else
+            memory_at_start -= memory_events[i].size;
+      }
+
+      paje.SetVariable( 0, variable_type_memory, container_memory, 1.0*memory_at_start/(1024*1024));
+
+      for(const auto & i : IntRange(n_memory_events_at_start, memory_events.size()))
+      {
+        auto & m = memory_events[i];
+        if(m.size==0)
+            continue;
+        double size = 1.0*m.size/(1024*1024);
+        if(m.is_alloc)
+          paje.AddVariable( m.time, variable_type_memory, container_memory, size);
+        else
+          paje.SubVariable( m.time, variable_type_memory, container_memory, size);
+      }
+
       std::set<int> timer_ids;
       std::map<int,int> timer_aliases;
+      std::map<int,std::string> timer_names;
 
       for(auto & event : timer_events)
-        timer_ids.insert(event.timer_id);
+          timer_ids.insert(event.timer_id);
 
+      for(auto & event : gpu_events)
+          timer_ids.insert(event.timer_id);
 
+      // Timer names
       for(auto & vtasks : tasks)
-        for (Task & t : vtasks)
-          if(t.id_type==Task::ID_TIMER)
-            timer_ids.insert(t.id);
+          for (Task & t : vtasks)
+              if(t.id_type==Task::ID_TIMER)
+                  timer_ids.insert(t.id);
 
       for(auto id : timer_ids)
-        timer_aliases[id] = paje.DefineEntityValue( state_type_timer, NgProfiler::GetName(id), -1 );
+          timer_names[id] = GetTimerName(id);
+
+      if(comm.Size()>1)
+      {
+        for(auto src : IntRange(0, comm.Size()))
+        {
+          if(src==MPI_PAJE_WRITER)
+            continue;
+
+          size_t n_timers;
+          comm.Recv (n_timers, src, 0);
+
+          int id;
+          std::string name;
+          for([[maybe_unused]]auto i : IntRange(n_timers))
+          {
+            comm.Recv (id, src, 0);
+            comm.Recv (name, src, 0);
+            timer_ids.insert(id);
+            timer_names[id] = name;
+          }
+        }
+      }
+
+      for(auto id : timer_ids)
+          timer_aliases[id] = paje.DefineEntityValue( state_type_timer, timer_names[id], -1 );
 
       int timerdepth = 0;
       int maxdepth = 0;
@@ -461,6 +668,54 @@ namespace ngcore
             paje.PopState( event.time, state_type_timer, timer_container_aliases[--timerdepth] );
         }
 
+      if(gpu_events.size())
+      {
+        auto gpu_container =  paje.CreateContainer( container_type_timer, container_task_manager, "GPU" );
+        for(auto & event : gpu_events)
+        {
+          if(event.is_start)
+            paje.PushState( event.time, state_type_timer, gpu_container, timer_aliases[event.timer_id] );
+          else
+            paje.PopState( event.time, state_type_timer, gpu_container);
+        }
+      }
+
+      if(user_events.size())
+        {
+          std::sort (user_events.begin(), user_events.end());
+
+          std::map<int, int> containers;
+
+          for(auto i : Range(user_containers.size()))
+          {
+              auto & [name, parent] = user_containers[i];
+              int a_parent = parent == -1 ? container_task_manager : containers[parent];
+              containers[i] = paje.CreateContainer( container_type_timer, a_parent, name );
+          }
+
+          for(auto ev : user_events)
+          {
+            if(containers[ev.container]==0)
+            {
+              std::string name = "User " + ToString(ev.container);
+              containers[ev.container] = paje.CreateContainer( container_type_timer, container_task_manager, name );
+            }
+          }
+
+          int i_start = 0;
+          for(auto i : Range(user_events.size()))
+          {
+            auto & event = user_events[i];
+            while(i_start < user_events.size() && user_events[i_start].t_start < event.t_end)
+            {
+              auto & ev = user_events[i_start];
+              paje.PushState( ev.t_start, state_type_timer, containers[ev.container], ev.data, ev.id );
+              i_start++;
+            }
+            paje.PopState( event.t_end, state_type_timer, containers[event.container]);
+          }
+        }
+
       for(auto & vtasks : tasks)
         {
           for (Task & t : vtasks) {
@@ -472,28 +727,80 @@ namespace ngcore
                   value_id = job_task_map[jobs[t.id-1].type];
                   if(trace_thread_counter)
                     {
-                      paje.AddVariable( t.start_time, variable_type_active_threads, container_jobs, 1.0 );
-                      paje.SubVariable( t.stop_time, variable_type_active_threads, container_jobs, 1.0 );
+                      if(t.is_start)
+                        paje.AddVariable( t.time, variable_type_active_threads, container_jobs, 1.0 );
+                      else
+                        paje.SubVariable( t.time, variable_type_active_threads, container_jobs, 1.0 );
                     }
                   if(trace_threads)
                     {
-                      paje.PushState( t.start_time, state_type_task, thread_aliases[t.thread_id], value_id, t.additional_value, true );
-                      paje.PopState( t.stop_time, state_type_task, thread_aliases[t.thread_id] );
+                      if(t.is_start)
+                        paje.PushState( t.time, state_type_task, thread_aliases[t.thread_id], value_id, t.additional_value, true );
+                      else
+                        paje.PopState( t.time, state_type_task, thread_aliases[t.thread_id] );
                     }
                   break;
                 case Task::ID_TIMER:
                   value_id = timer_aliases[t.id];
-                  paje.PushState( t.start_time, state_type_timer, thread_aliases[t.thread_id], value_id, t.additional_value, true );
-                  paje.PopState( t.stop_time, state_type_timer, thread_aliases[t.thread_id] );
+                  if(t.is_start)
+                    paje.PushState( t.time, state_type_timer, thread_aliases[t.thread_id], value_id, t.additional_value, true );
+                  else
+                    paje.PopState( t.time, state_type_timer, thread_aliases[t.thread_id] );
                   break;
                 default:
-                  paje.PushState( t.start_time, state_type_task, thread_aliases[t.thread_id], value_id, t.additional_value, false );
-                  paje.PopState( t.stop_time, state_type_task, thread_aliases[t.thread_id] );
+                  if(t.is_start)
+                    paje.PushState( t.time, state_type_task, thread_aliases[t.thread_id], value_id, t.additional_value, false );
+                  else
+                    paje.PopState( t.time, state_type_task, thread_aliases[t.thread_id] );
                   break;
                 }
           }
         }
 
+#ifdef PARALLEL
+      if(comm.Size()>1)
+      {
+        for(auto & event : timer_events)
+        {
+          if(event.is_start)
+            paje.PushState( event.time, state_type_timer, thread_aliases[MPI_PAJE_WRITER], timer_aliases[event.timer_id] );
+          else
+            paje.PopState( event.time, state_type_timer, thread_aliases[MPI_PAJE_WRITER] );
+        }
+
+        // Timer events
+        Array<int> timer_id;
+        Array<TTimePoint> time;
+        Array<bool> is_start;
+        Array<int> thread_id;
+
+        for(auto src : IntRange(0, comm.Size()))
+        {
+          if(src==MPI_PAJE_WRITER)
+            continue;
+
+          comm.Recv (timer_id, src, 0);
+          comm.Recv (time, src, 0);
+          comm.Recv (is_start, src, 0);
+          comm.Recv (thread_id, src, 0);
+
+          for(auto i : Range(timer_id.Size()))
+          {
+            TimerEvent event;
+            event.timer_id = timer_id[i];
+            event.time = time[i];
+            event.is_start = is_start[i];
+            event.thread_id = thread_id[i];
+
+            if(event.is_start)
+              paje.PushState( event.time, state_type_timer, thread_aliases[src], timer_aliases[event.timer_id] );
+            else
+              paje.PopState( event.time, state_type_timer, thread_aliases[src] );
+          }
+        }
+      }
+#endif // PARALLEL
+
       // Merge link event
       int nlinks = 0;
       for( auto & l : links)
@@ -552,25 +859,92 @@ namespace ngcore
                 }
             }
         }
-      WriteSunburstHTML();
       paje.WriteEvents();
     }
 
+  void PajeTrace::SendData( )
+    {
+#ifdef PARALLEL
+      // Hostname
+      NgMPI_Comm comm(NG_MPI_COMM_WORLD);
+      // auto rank = comm.Rank();
+      // auto nranks = comm.Size();
+
+      std::string hostname;
+        {
+          std::array<char, ASSUMED_MPI_MAX_PROCESSOR_NAME+1> ahostname;
+          int len;
+          NG_MPI_Get_processor_name(ahostname.data(), &len);
+          hostname = ahostname.data();
+        }
+
+      comm.Send(hostname, MPI_PAJE_WRITER, 0);
+
+      // Timer names
+      std::set<int> timer_ids;
+      std::map<int,std::string> timer_names;
+
+      for(auto & event : timer_events)
+          timer_ids.insert(event.timer_id);
+
+      for(auto id : timer_ids)
+          timer_names[id] = GetTimerName(id);
+      size_t size = timer_ids.size();
+      comm.Send(size, MPI_PAJE_WRITER, 0);
+      for(auto id : timer_ids)
+        {
+          comm.Send(id, MPI_PAJE_WRITER, 0);
+          comm.Send(timer_names[id], MPI_PAJE_WRITER, 0);
+        }
+
+
+      // Timer events
+      Array<int> timer_id;
+      Array<TTimePoint> time;
+      Array<bool> is_start;
+      Array<int> thread_id;
+
+      for(auto & event : timer_events)
+        {
+          timer_id.Append(event.timer_id);
+          time.Append(event.time);
+          is_start.Append(event.is_start);
+          thread_id.Append(event.thread_id);
+        }
+
+      comm.Send (timer_id, MPI_PAJE_WRITER, 0);
+      comm.Send (time, MPI_PAJE_WRITER, 0);
+      comm.Send (is_start, MPI_PAJE_WRITER, 0);
+      comm.Send (thread_id, MPI_PAJE_WRITER, 0);
+#endif // PARALLEL
+    }
+
   ///////////////////////////////////////////////////////////////////
   // Write HTML file drawing a sunburst chart with cumulated timings
   struct TreeNode
   {
       int id = 0;
       std::map<int, TreeNode> children;
-      double time = 0.0;
+      double chart_size = 0.0; // time without children (the chart lib accumulates children sizes again)
+      double size = 0.0;
+      double min_size = 1e99;
+      double max_size = 0.0;
       std::string name;
+
+      size_t calls = 0;
       TTimePoint start_time = 0;
   };
 
-  void PrintNode (const TreeNode &n, int &level, std::ofstream & f);
-  void PrintNode (const TreeNode &n, int &level, std::ofstream & f)
+  void PrintNode (const TreeNode &n, std::ofstream & f)
   {
-      f << "{ name: \"" + n.name + "\", size: " + ToString(n.time);
+      f << "{ name: \"" + n.name + "\"";
+      f << ", calls: " << n.calls;
+      f << ", size: " << n.chart_size;
+      f << ", value: " << n.size;
+      f << ", min: " << n.min_size;
+      f << ", max: " << n.max_size;
+      if(n.calls)
+        f << ", avg: " << n.size/n.calls;
       int size = n.children.size();
       if(size>0)
       {
@@ -578,7 +952,7 @@ namespace ngcore
           f << ", children: [";
           for(auto & c : n.children)
           {
-              PrintNode(c.second, level, f);
+              PrintNode(c.second, f);
               if(++i<size)
                   f << " , ";
           }
@@ -587,12 +961,232 @@ namespace ngcore
       f << '}';
   }
 
-  void PajeTrace::WriteSunburstHTML( )
+  void WriteSunburstHTML( TreeNode & root, std::string filename, bool time_or_memory )
+  {
+    std::ofstream f(filename+".html");
+    f.precision(4);
+    f << R"CODE_(
+<head>
+  <script src="https://cdn.jsdelivr.net/npm/d3@7"></script>
+  <script src="https://unpkg.com/sunburst-chart"></script>
+
+  <style>
+    body { margin: 0 }
+    .tooltip {
+      white-space: pre-line !important;
+      max-width: 800px !important;
+      word-wrap: break-word !important;
+      padding: 10px !important;
+    }
+  </style>
+)CODE_";
+    if(!time_or_memory)
+      f << "<title>Maximum Memory Consumption</title>\n";
+    f << R"CODE_(
+</head>
+<body>
+  <div id="chart"></div>
+
+  <script>
+    const data = 
+)CODE_";
+      PrintNode(root, f);
+      f << ";\n\n";
+      if(time_or_memory)
+        f << "const chart_type = 'time';\n";
+      else
+        f << "const chart_type = 'memory';\n";
+      f << R"CODE_(
+    const color = d3.scaleOrdinal(d3.schemePaired);
+
+    let getTime = (t) =>
+    {
+       if(t>=1000)  return (t/1000).toPrecision(4) + '  s';
+       if(t>=0.1)   return t.toPrecision(4) + ' ms';
+       if(t>=1e-4)  return (t*1e3).toPrecision(4) + ' us';
+
+       return (t/1e6).toPrecision(4) + ' ns';
+    };
+
+    const KB_ = 1024;
+    const MB_ = KB_*1024;
+    const GB_ = MB_*1024;
+    let getMemory = (m) =>
+    {
+       if(m>=GB_)  return (m/GB_).toPrecision(4) + ' GB';
+       if(m>=MB_)  return (m/MB_).toPrecision(4) + ' MB';
+       if(m>=KB_)  return (m/KB_).toPrecision(4) + ' KB';
+       return m.toPrecision(4) + ' B';
+    };
+
+    Sunburst()
+      .data(data)
+      .size('size')
+      .color(d => color(d.name))
+      .tooltipTitle((d, node) => { return node.parent ? node.parent.data.name + " &rarr; " + d.name : d.name; })
+      .tooltipContent((d, node) => {
+        if(chart_type=="memory")
+        {
+          return `Total Memory: <i>${getMemory(d.value)}</i> <br>`
+               + `Memory: <i>${getMemory(d.size)}</i>`
+        }
+        else
+        {
+          return `Time: <i>${getTime(d.value)}</i> <br>`
+               + `calls: <i>${d.calls}</i> <br>`
+               + `min: <i>${getTime(d.min)}</i> <br>`
+               + `max: <i>${getTime(d.max)}</i> <br>`
+               + `avg: <i>${getTime(d.avg)}</i>`
+        }
+      })
+      (document.getElementById('chart'));
+
+      // Line breaks in tooltip
+      var all = document.getElementsByClassName('sunbirst-tooltip');
+      for (var i = 0; i < all.length; i++) {
+          all[i].white_space = "";
+      }
+  </script>
+</body>
+)CODE_" << std::endl;
+
+
+  }
+
+#ifdef NETGEN_TRACE_MEMORY
+  void PajeTrace::WriteMemoryChart( std::string fname )
+  {
+    if(fname=="")
+      fname = tracefile_name + "_memory";
+    size_t mem_allocated = 0;
+    size_t max_mem_allocated = 0;
+    size_t imax_mem_allocated = 0;
+
+    const auto & names = MemoryTracer::GetNames();
+    const auto & parents = MemoryTracer::GetParents();
+    size_t N = names.size();
+
+    Array<size_t> mem_allocated_id;
+    mem_allocated_id.SetSize(N);
+    mem_allocated_id = 0;
+
+    // Find point with maximum memory allocation, check for missing allocs/frees
+    for(auto i : IntRange(memory_events.size()))
+    {
+      const auto & ev = memory_events[i];
+
+      if(ev.is_alloc)
+      {
+        mem_allocated += ev.size;
+        mem_allocated_id[ev.id] += ev.size;
+        if(mem_allocated > max_mem_allocated && i>=n_memory_events_at_start)
+        {
+          imax_mem_allocated = i;
+          max_mem_allocated = mem_allocated;
+        }
+      }
+      else
+      {
+        if(ev.size > mem_allocated)
+          {
+            std::cerr << "Error in memory tracer: have total allocated memory < 0" << std::endl;
+            mem_allocated = 0;
+          }
+        else
+          mem_allocated -= ev.size;
+        if(ev.size > mem_allocated_id[ev.id])
+          {
+            std::cerr << "Error in memory tracer: have allocated memory < 0 in tracer " << names[ev.id] << std::endl;
+            mem_allocated_id[ev.id] = 0;
+          }
+        else
+          mem_allocated_id[ev.id] -= ev.size;
+      }
+    }
+
+    // reconstruct again the memory consumption after event imax_mem_allocated
+    mem_allocated_id = 0;
+    for(auto i : IntRange(imax_mem_allocated+1))
+    {
+      const auto & ev = memory_events[i];
+
+      if(ev.is_alloc)
+        mem_allocated_id[ev.id] += ev.size;
+      else
+        {
+          if(ev.size > mem_allocated_id[ev.id])
+            mem_allocated_id[ev.id] = 0;
+          else
+            mem_allocated_id[ev.id] -= ev.size;
+        }
+    }
+
+    TreeNode root;
+    root.name="all";
+
+    Array<TreeNode*> nodes;
+    nodes.SetSize(N);
+    nodes = nullptr;
+    nodes[0] = &root;
+    Array<Array<int>> children(N);
+
+    Array<size_t> sorting; // topological sorting (parents before children)
+    sorting.SetAllocSize(N);
+
+    for(auto i : IntRange(1, N))
+        children[parents[i]].Append(i);
+
+    ArrayMem<size_t, 100> stack;
+    sorting.Append(0);
+    stack.Append(0);
+
+    while(stack.Size())
+    {
+      auto current = stack.Last();
+      stack.DeleteLast();
+
+      for(const auto child : children[current])
+      {
+        sorting.Append(child);
+        if(children[child].Size())
+          stack.Append(child);
+      }
+    }
+
+    for(auto i : sorting)
+    {
+      if(i==0)
+          continue;
+
+      TreeNode * parent = nodes[parents[i]];
+
+      auto & node = parent->children[i];
+      nodes[i] = &node;
+      node.id = i;
+      node.chart_size = mem_allocated_id[i];
+      node.size = mem_allocated_id[i];
+      node.name = names[i];
+    }
+
+    for(auto i_ : Range(sorting))
+    {
+      // reverse topological order to accumulate total memory usage of all children
+      auto i = sorting[sorting.Size()-1-i_];
+      if(i==0)
+          continue;
+      nodes[parents[i]]->size += nodes[i]->size;
+    }
+
+    WriteSunburstHTML( root, fname, false );
+
+  }
+#endif // NETGEN_TRACE_MEMORY
+
+  void PajeTrace::WriteTimingChart( )
   {
       std::vector<TimerEvent> events;
 
       TreeNode root;
-      root.time=0;
       root.name="all";
       TreeNode *current = &root;
 
@@ -629,7 +1223,10 @@ namespace ngcore
 
       std::sort (events.begin(), events.end());
 
-      root.time = 1000.0*static_cast<double>(stop_time-start_time) * seconds_per_tick;
+      root.size = 1000.0*static_cast<double>(stop_time) * seconds_per_tick;
+      root.calls = 1;
+      root.min_size = root.size;
+      root.max_size = root.size;
 
       for(auto & event : events)
       {
@@ -645,8 +1242,8 @@ namespace ngcore
 
               if(need_init)
               {
-                  current->name = is_timer_event ? NgProfiler::GetName(id) : job_names[id];
-                  current->time = 0.0;
+                  current->name = is_timer_event ? GetTimerName(id) : job_names[id];
+                  current->size = 0.0;
                   current->id = id;
               }
 
@@ -658,44 +1255,22 @@ namespace ngcore
                 std::cout << "node stack empty!" << std::endl;
                 break;
               }
-              double time = 1000.0*static_cast<double>(event.time-current->start_time) * seconds_per_tick;
-              current->time += time;
+              double size = 1000.0*static_cast<double>(event.time-current->start_time) * seconds_per_tick;
+              current->size += size;
+              current->chart_size += size;
+              current->min_size = std::min(current->min_size, size);
+              current->max_size = std::max(current->max_size, size);
+              current->calls++;
+
               current = node_stack.back();
-              current->time -= time;
+              current->chart_size -= size;
               node_stack.pop_back();
           }
       }
 
-      int level = 0;
-      std::ofstream f(tracefile_name+".html");
-      f.precision(4);
-      f << R"CODE_(
-<head>
-  <script src="https://d3js.org/d3.v5.min.js"></script>
-  <script src="https://unpkg.com/sunburst-chart"></script>
+      root.chart_size = 0.0;
 
-  <style>body { margin: 0 }</style>
-</head>
-<body>
-  <div id="chart"></div>
-
-  <script>
-    const data = 
-)CODE_";
-      PrintNode(root, level, f);
-      f << R"CODE_( ;
-
-    const color = d3.scaleOrdinal(d3.schemePaired);
-
-    Sunburst()
-      .data(data)
-      .size('size')
-      .color(d => color(d.name))
-      .tooltipContent((d, node) => `Time: <i>${node.value.toPrecision(6)}ms</i>`)
-      (document.getElementById('chart'));
-  </script>
-</body>
-)CODE_" << std::endl;
+      ngcore::WriteSunburstHTML( root, tracefile_name, true );
   }
 
 } // namespace ngcore

libsrc/core/paje_trace.hpp

@@ -1,6 +1,7 @@
 #ifndef NETGEN_CORE_PAJE_TRACE_HPP
 #define NETGEN_CORE_PAJE_TRACE_HPP
 
+#include <algorithm>
 #include <limits>
 #include <vector>
 
@@ -23,18 +24,31 @@ namespace ngcore
       NGCORE_API static size_t max_tracefile_size;
       NGCORE_API static bool trace_thread_counter;
       NGCORE_API static bool trace_threads;
+      NGCORE_API static bool mem_tracing_enabled;
+      NGCORE_API static bool write_paje_file;
 
       bool tracing_enabled;
       TTimePoint start_time;
       int nthreads;
+      size_t n_memory_events_at_start;
 
     public:
-      void WriteSunburstHTML();
+      NGCORE_API void Write();
+      NGCORE_API void WritePajeFile( const std::string & filename );
+      NGCORE_API void WriteTimingChart();
+#ifdef NETGEN_TRACE_MEMORY
+      NGCORE_API void WriteMemoryChart( std::string fname );
+#endif // NETGEN_TRACE_MEMORY
 
       // Approximate number of events to trace. Tracing will
       // be stopped if any thread reaches this number of events
       unsigned int max_num_events_per_thread;
 
+      static void SetTraceMemory( bool trace_memory )
+        {
+          mem_tracing_enabled = trace_memory;
+        }
+
       static void SetTraceThreads( bool atrace_threads )
         {
           trace_threads = atrace_threads;
@@ -50,6 +64,11 @@ namespace ngcore
           max_tracefile_size = max_size;
         }
 
+      static void SetWritePajeFile( bool write )
+        {
+          write_paje_file = write;
+        }
+
       std::string tracefile_name;
 
       struct Job
@@ -69,8 +88,8 @@ namespace ngcore
 
           int additional_value;
 
-          TTimePoint start_time;
-          TTimePoint stop_time;
+          TTimePoint time;
+          bool is_start;
 
           static constexpr int ID_NONE = -1;
           static constexpr int ID_JOB = 1;
@@ -87,6 +106,16 @@ namespace ngcore
           bool operator < (const TimerEvent & other) const { return time < other.time; }
         };
 
+      struct UserEvent
+        {
+          TTimePoint t_start = 0, t_end = 0;
+          std::string data = "";
+          int container = 0;
+          int id = 0;
+
+          bool operator < (const UserEvent & other) const { return t_start < other.t_start; }
+        };
+
       struct ThreadLink
         {
           int thread_id;
@@ -96,10 +125,24 @@ namespace ngcore
           bool operator < (const ThreadLink & other) const { return time < other.time; }
         };
 
+      struct MemoryEvent
+        {
+          TTimePoint time;
+          size_t size;
+          int id;
+          bool is_alloc;
+
+          bool operator < (const MemoryEvent & other) const { return time < other.time; }
+        };
+
       std::vector<std::vector<Task> > tasks;
       std::vector<Job> jobs;
       std::vector<TimerEvent> timer_events;
+      std::vector<UserEvent> user_events;
+      std::vector<std::tuple<std::string, int>> user_containers;
+      std::vector<TimerEvent> gpu_events;
       std::vector<std::vector<ThreadLink> > links;
+      NGCORE_API static std::vector<MemoryEvent> memory_events;
 
     public:
       NGCORE_API void StopTracing();
@@ -113,6 +156,36 @@ namespace ngcore
       void operator=(const PajeTrace &) = delete;
       void operator=(PajeTrace &&) = delete;
 
+      int AddUserContainer(std::string name, int parent=-1)
+      {
+        if(auto pos = std::find(user_containers.begin(), user_containers.end(), std::tuple{name,parent}); pos != user_containers.end())
+          return pos - user_containers.begin();
+        int id = user_containers.size();
+        user_containers.push_back({name, parent});
+        return id;
+      }
+
+      void AddUserEvent(UserEvent ue)
+      {
+          if(!tracing_enabled) return;
+          user_events.push_back(ue);
+      }
+      void StartGPU(int timer_id = 0)
+        {
+          if(!tracing_enabled) return;
+          if(unlikely(gpu_events.size() == max_num_events_per_thread))
+            StopTracing();
+          gpu_events.push_back(TimerEvent{timer_id, GetTimeCounter(), true});
+        }
+
+      void StopGPU(int timer_id)
+        {
+          if(!tracing_enabled) return;
+          if(unlikely(gpu_events.size() == max_num_events_per_thread))
+            StopTracing();
+          gpu_events.push_back(TimerEvent{timer_id, GetTimeCounter(), false});
+        }
+
       void StartTimer(int timer_id)
         {
           if(!tracing_enabled) return;
@@ -129,30 +202,44 @@ namespace ngcore
           timer_events.push_back(TimerEvent{timer_id, GetTimeCounter(), false});
         }
 
-      NETGEN_INLINE int StartTask(int thread_id, int id, int id_type = Task::ID_NONE, int additional_value = -1)
+      void AllocMemory(int id, size_t size)
+        {
+          if(!mem_tracing_enabled) return;
+          memory_events.push_back(MemoryEvent{GetTimeCounter(), size, id, true});
+        }
+
+      void FreeMemory(int id, size_t size)
+        {
+          if(!mem_tracing_enabled) return;
+          memory_events.push_back(MemoryEvent{GetTimeCounter(), size, id, false});
+        }
+
+      void ChangeMemory(int id, long long size)
+        {
+          if(size>0)
+            AllocMemory(id, size);
+          if(size<0)
+            FreeMemory(id, -size);
+        }
+
+
+      int StartTask(int thread_id, int id, int id_type = Task::ID_NONE, int additional_value = -1)
         {
           if(!tracing_enabled) return -1;
           if(!trace_threads && !trace_thread_counter) return -1;
 	  if(unlikely(tasks[thread_id].size() == max_num_events_per_thread))
             StopTracing();
           int task_num = tasks[thread_id].size();
-          tasks[thread_id].push_back( Task{thread_id, id, id_type, additional_value, GetTimeCounter()} );
+          tasks[thread_id].push_back( Task{thread_id, id, id_type, additional_value, GetTimeCounter(), true} );
           return task_num;
         }
 
-      void StopTask(int thread_id, int task_num)
+      void StopTask(int thread_id, int id, int id_type = Task::ID_NONE)
         {
           if(!trace_threads && !trace_thread_counter) return;
-          if(task_num>=0)
-            tasks[thread_id][task_num].stop_time = GetTimeCounter();
+          tasks[thread_id].push_back( Task{thread_id, id, id_type, 0, GetTimeCounter(), false} );
         }
 
-      void SetTask(int thread_id, int task_num, int additional_value) {
-          if(!trace_threads && !trace_thread_counter) return;
-          if(task_num>=0)
-            tasks[thread_id][task_num].additional_value = additional_value;
-      }
-
       void StartJob(int job_id, const std::type_info & type)
         {
           if(!tracing_enabled) return;
@@ -183,7 +270,7 @@ namespace ngcore
           links[thread_id].push_back( ThreadLink{thread_id, key, GetTimeCounter(), false} );
         }
 
-      void Write( const std::string & filename );
+      void SendData(); // MPI parallel data reduction
 
     };
 } // namespace ngcore

libsrc/core/profiler.cpp

@@ -94,7 +94,7 @@ namespace ngcore
     if (first_overflow)
       {
         first_overflow = false;
-        NgProfiler::logger->warn("no more timer available, reusing last one");
+        NgProfiler::logger->warn( ("no more timer available ("+name+"), reusing last one").c_str());
       }
     return 0;
   }
@@ -113,5 +113,10 @@ namespace ngcore
 
   NgProfiler prof; // NOLINT
 
+#ifdef NETGEN_TRACE_MEMORY
+  std::vector<std::string> MemoryTracer::names{"all"};
+  std::vector<int> MemoryTracer::parents{-1};
+  std::atomic<size_t> MemoryTracer::total_memory{0};
+#endif // NETGEN_TRACE_MEMORY
 
 } // namespace ngcore

libsrc/core/profiler.hpp

@@ -3,10 +3,13 @@
 
 #include <array>
 #include <chrono>
+#include <functional>
 #include <string>
 
+#include "array.hpp"
 #include "logging.hpp"
 #include "paje_trace.hpp"
+#include "taskmanager.hpp"
 #include "utils.hpp"
 
 namespace ngcore
@@ -144,39 +147,98 @@ namespace ngcore
     };
   };
 
+  
+  struct TNoTracing{ static constexpr bool do_tracing=false; };
+  struct TTracing{ static constexpr bool do_tracing=true; };
 
+  struct TNoTiming{ static constexpr bool do_timing=false; };
+  struct TTiming{ static constexpr bool do_timing=true; };
 
-  class NGCORE_API Timer
+  namespace detail {
+
+      template<typename T>
+      constexpr bool is_tracing_type_v = std::is_same_v<T, TNoTracing> || std::is_same_v<T, TTracing>;
+
+      template<typename T>
+      constexpr bool is_timing_type_v = std::is_same_v<T, TNoTiming> || std::is_same_v<T, TTiming>;
+  }
+
+  [[maybe_unused]] static TNoTracing NoTracing;
+  [[maybe_unused]] static TNoTiming NoTiming;
+
+  template<typename TTracing=TTracing, typename TTiming=TTiming>
+  class Timer
   {
     int timernr;
-    int priority;
-  public:
-    Timer (const std::string & name, int apriority = 1)
-      : priority(apriority)
+    int Init( const std::string & name )
     {
-      timernr = NgProfiler::CreateTimer (name);
+      return NgProfiler::CreateTimer (name);
     }
+  public:
+    static constexpr bool do_tracing = TTracing::do_tracing;
+    static constexpr bool do_timing = TTiming::do_timing;
+
+    Timer (const std::string & name) : timernr(Init(name)) { }
+
+    template<std::enable_if_t< detail::is_tracing_type_v<TTracing>, bool> = false>
+    Timer( const std::string & name, TTracing ) : timernr(Init(name)) { }
+
+    template<std::enable_if_t< detail::is_timing_type_v<TTiming>, bool> = false>
+    Timer( const std::string & name, TTiming ) : timernr(Init(name)) { }
+
+    Timer( const std::string & name, TTracing, TTiming ) : timernr(Init(name)) { }
+
+    [[deprecated ("Use Timer(name, NoTracing/NoTiming) instead")]] Timer( const std::string & name, int ) : timernr(Init(name)) {}
+
     void SetName (const std::string & name)
     {
       NgProfiler::SetName (timernr, name);
     }
-    void Start ()
+    void Start () const
     {
-      if (priority <= 2)
-	NgProfiler::StartTimer (timernr);
-      if (priority <= 1)
-        if(trace) trace->StartTimer(timernr);
+      Start(TaskManager::GetThreadId());
     }
-    void Stop ()
+    void Stop () const
     {
-      if (priority <= 2)
-	NgProfiler::StopTimer (timernr);
-      if (priority <= 1)
-        if(trace) trace->StopTimer(timernr);
+      Stop(TaskManager::GetThreadId());
+    }
+    void Start (int tid) const
+    {
+        if(tid==0)
+        {
+          if constexpr(do_timing)
+            NgProfiler::StartTimer (timernr);
+          if constexpr(do_tracing)
+            if(trace) trace->StartTimer(timernr);
+        }
+        else
+        {
+          if constexpr(do_timing)
+            NgProfiler::StartThreadTimer(timernr, tid);
+          if constexpr(do_tracing)
+            if(trace) trace->StartTask (tid, timernr, PajeTrace::Task::ID_TIMER);
+        }
+    }
+    void Stop (int tid) const
+    {
+        if(tid==0)
+        {
+            if constexpr(do_timing)
+                NgProfiler::StopTimer (timernr);
+            if constexpr(do_tracing)
+                if(trace) trace->StopTimer(timernr);
+        }
+        else
+        {
+          if constexpr(do_timing)
+            NgProfiler::StopThreadTimer(timernr, tid);
+          if constexpr(do_tracing)
+            if(trace) trace->StopTask (tid, timernr, PajeTrace::Task::ID_TIMER);
+        }
     }
     void AddFlops (double aflops)
     {
-      if (priority <= 2)
+      if constexpr(do_timing)
 	NgProfiler::AddFlops (timernr, aflops);
     }
 
@@ -185,7 +247,7 @@ namespace ngcore
     double GetMFlops ()
     { return NgProfiler::GetFlops(timernr)
         / NgProfiler::GetTime(timernr) * 1e-6; }
-    operator int () { return timernr; }
+    operator int () const { return timernr; }
   };
 
 
@@ -193,14 +255,21 @@ namespace ngcore
      Timer object.
        Start / stop timer at constructor / destructor.
   */
+  template<typename TTimer>
   class RegionTimer
   {
-    Timer & timer;
+    const TTimer & timer;
+    int tid;
   public:
     /// start timer
-    RegionTimer (Timer & atimer) : timer(atimer) { timer.Start(); }
+    RegionTimer (const TTimer & atimer) : timer(atimer)
+    {
+      tid = TaskManager::GetThreadId();
+      timer.Start(tid);
+    }
+
     /// stop timer
-    ~RegionTimer () { timer.Stop(); }
+    ~RegionTimer () { timer.Stop(tid); }
 
     RegionTimer() = delete;
     RegionTimer(const RegionTimer &) = delete;
@@ -209,7 +278,7 @@ namespace ngcore
     void operator=(RegionTimer &&) = delete;
   };
 
-  class ThreadRegionTimer
+  class [[deprecated("Use RegionTimer instead (now thread safe)")]] ThreadRegionTimer
   {
     size_t nr;
     size_t tid;
@@ -232,6 +301,7 @@ namespace ngcore
     {
       int nr;
       int thread_id;
+      int type;
     public:
       static constexpr int ID_JOB = PajeTrace::Task::ID_JOB;
       static constexpr int ID_NONE = PajeTrace::Task::ID_NONE;
@@ -248,28 +318,26 @@ namespace ngcore
         : thread_id(athread_id)
         {
 	  if (trace)
-          nr = trace->StartTask (athread_id, region_id, id_type, additional_value);
+          trace->StartTask (athread_id, region_id, id_type, additional_value);
+          type = id_type;
+          nr = region_id;
         }
       /// start trace with timer
-      RegionTracer (int athread_id, Timer & timer, int additional_value = -1 )
+      template<typename TTimer>
+      RegionTracer (int athread_id, TTimer & timer, int additional_value = -1 )
         : thread_id(athread_id)
         {
+          nr = timer;
+          type = ID_TIMER;
 	  if (trace)
-          nr = trace->StartTask (athread_id, static_cast<int>(timer), ID_TIMER, additional_value);
+            trace->StartTask (athread_id, nr, type, additional_value);
         }
 
-      /// set user defined value
-      void SetValue( int additional_value )
-      {
-	  if (trace)
-        trace->SetTask( thread_id, nr, additional_value );
-      }
-
       /// stop trace
       ~RegionTracer ()
         {
 	  if (trace)
-          trace->StopTask (thread_id, nr);
+            trace->StopTask (thread_id, nr, type);
         }
     };
 
@@ -301,5 +369,14 @@ namespace ngcore
 
 } // namespace ngcore
 
+// Helper macro to easily add multiple timers in a function for profiling
+// Usage: NETGEN_TIMER_FROM_HERE("my_timer_name")
+// Effect: define static Timer and RegionTimer with given name and line number
+#define NETGEN_TOKEN_CONCAT(x, y) x ## y
+#define NETGEN_TOKEN_CONCAT2(x, y) NETGEN_TOKEN_CONCAT(x, y)
+#define NETGEN_TIMER_FROM_HERE(name) \
+  static Timer NETGEN_TOKEN_CONCAT2(timer_, __LINE__)( string(name)+"_"+ToString(__LINE__)); \
+  RegionTimer NETGEN_TOKEN_CONCAT2(rt_,__LINE__)(NETGEN_TOKEN_CONCAT2(timer_,__LINE__));
+
 
 #endif // NETGEN_CORE_PROFILER_HPP

libsrc/core/python_ngcore.cpp

@@ -8,17 +8,17 @@ using std::string;
 namespace ngcore
 {
   bool ngcore_have_numpy = false;
+  bool parallel_pickling = true;
+  
   void SetFlag(Flags &flags, string s, py::object value) 
   {
     if (py::isinstance<py::dict>(value))
       {             
-        py::dict vdd(value);
-        // call recursively to set dictionary
-        for (auto item : vdd) {
-          string name = item.first.cast<string>();
-          py::object val = py::reinterpret_borrow<py::object>(item.second);
-          SetFlag(flags, name, val);
-        }
+        Flags nested_flags;
+        for(auto item : value.cast<py::dict>())
+          SetFlag(nested_flags, item.first.cast<string>(),
+                  item.second.cast<py::object>());
+        flags.SetFlag(s, nested_flags);
         return;
       }
 
@@ -101,7 +101,9 @@ namespace ngcore
           }
       }
 
-    auto flags = py::cast<Flags>(flags_dict);
+    Flags flags;
+    for(auto item : flags_dict)
+      SetFlag(flags, item.first.cast<string>(), item.second.cast<py::object>());
 
     for (auto item : kwargs)
       {
@@ -155,6 +157,11 @@ namespace ngcore
         auto val = flags.GetDefineFlag(i, key);
         d[key.c_str()] = val;
       }
+    for(auto i : Range(flags.GetNAnyFlags()))
+      {
+        auto& a = flags.GetAnyFlag(i, key);
+        d[key.c_str()] = CastAnyToPy(a);
+      }
     return d;
   }
 

libsrc/core/python_ngcore.hpp

@@ -5,17 +5,145 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/operators.h>
 #include <pybind11/numpy.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
 
 #include "array.hpp"
+#include "table.hpp"
 #include "archive.hpp"
 #include "flags.hpp"
 #include "ngcore_api.hpp"
+#include "ng_mpi.hpp"
+
 namespace py = pybind11;
 
 namespace ngcore
 {
-  NGCORE_API extern bool ngcore_have_numpy;
+#ifdef PARALLEL
+  NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &);
+  NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm);
+#endif // PARALLEL
 
+  namespace detail
+  {
+    template<typename T>
+    struct HasPyFormat
+    {
+    private:
+      template<typename T2>
+      static auto check(T2*) -> std::enable_if_t<std::is_same_v<decltype(std::declval<py::format_descriptor<T2>>().format()), std::string>, std::true_type>;
+      static auto check(...) -> std::false_type;
+    public:
+      static constexpr bool value = decltype(check((T*) nullptr))::value;
+    };
+  } // namespace detail
+
+#ifdef PARALLEL
+  struct mpi4py_comm {
+    mpi4py_comm() = default;
+    mpi4py_comm(NG_MPI_Comm value) : value(value) {}
+    operator NG_MPI_Comm () { return value; }
+
+    NG_MPI_Comm value;
+  };
+#endif  // PARALLEL
+} // namespace ngcore
+
+
+////////////////////////////////////////////////////////////////////////////////
+// automatic conversion of python list to Array<>
+namespace pybind11 {
+namespace detail {
+
+#ifdef PARALLEL
+template <> struct type_caster<ngcore::mpi4py_comm> {
+  public:
+  PYBIND11_TYPE_CASTER(ngcore::mpi4py_comm, _("mpi4py_comm"));
+
+    // Python -> C++
+    bool load(handle src, bool) {
+      return ngcore::NG_MPI_CommFromMPI4Py(src, value.value);
+    }
+
+    // C++ -> Python
+    static handle cast(ngcore::mpi4py_comm src,
+                       return_value_policy /* policy */,
+                       handle /* parent */)
+    {
+      // Create an mpi4py handle
+      return ngcore::NG_MPI_CommToMPI4Py(src.value);
+    }
+};
+#endif //  PARALLEL
+
+template <typename Type, typename Value> struct ngcore_list_caster {
+    using value_conv = make_caster<Value>;
+
+    bool load(handle src, bool convert) {
+        if (!isinstance<sequence>(src) || isinstance<str>(src))
+            return false;
+        auto s = reinterpret_borrow<sequence>(src);
+        value.SetSize(s.size());
+        value.SetSize0();
+        for (auto it : s) {
+            value_conv conv;
+            if (!conv.load(it, convert))
+                return false;
+            value.Append(cast_op<Value &&>(std::move(conv)));
+        }
+        return true;
+    }
+
+public:
+    template <typename T>
+    static handle cast(T &&src, return_value_policy policy, handle parent) {
+        if (!std::is_lvalue_reference<T>::value)
+            policy = return_value_policy_override<Value>::policy(policy);
+        list l(src.Size());
+        size_t index = 0;
+        for (auto &&value : src) {
+            auto value_ = reinterpret_steal<object>(value_conv::cast(forward_like<T>(value), policy, parent));
+            if (!value_)
+                return handle();
+            PyList_SET_ITEM(l.ptr(), (ssize_t) index++, value_.release().ptr()); // steals a reference
+        }
+        return l.release();
+    }
+
+    PYBIND11_TYPE_CASTER(Type, _("Array[") + value_conv::name + _("]"));
+};
+
+
+template <typename Type> struct type_caster<ngcore::Array<Type>, enable_if_t<!ngcore::detail::HasPyFormat<Type>::value>>
+ : ngcore_list_caster<ngcore::Array<Type>, Type> { };
+
+
+  /*
+  template <typename Type> struct type_caster<std::shared_ptr<ngcore::Table<Type>>>
+  {
+    template <typename T>
+    static handle cast(T &&src, return_value_policy policy, handle parent)
+    {
+      std::cout << "handle called with type src = " << typeid(src).name() << std::endl;
+
+      return handle(); // what so ever
+    }
+    
+    PYBIND11_TYPE_CASTER(Type, _("Table[") + make_caster<Type>::name + _("]"));
+  };
+  */
+  
+  
+
+} // namespace detail
+} // namespace pybind11
+////////////////////////////////////////////////////////////////////////////////
+
+namespace ngcore
+{
+  NGCORE_API extern bool ngcore_have_numpy;
+  NGCORE_API extern bool parallel_pickling;
+  
   // Python class name type traits
   template <typename T>
   struct PyNameTraits {
@@ -65,6 +193,93 @@ namespace ngcore
     { return std::string("sp_")+GetPyName<T>(); }
   };
 
+  template<typename ARCHIVE>
+  class NGCORE_API_EXPORT PyArchive : public ARCHIVE
+  {
+  private:
+    pybind11::list lst;
+    size_t index = 0;
+    std::map<std::string, VersionInfo> version_needed;
+  protected:
+    using ARCHIVE::stream;
+    using ARCHIVE::version_map;
+  public:
+    PyArchive(const pybind11::object& alst = pybind11::none()) :
+      ARCHIVE(std::make_shared<std::stringstream>()),
+      lst(alst.is_none() ? pybind11::list() : pybind11::cast<pybind11::list>(alst))
+    {
+      ARCHIVE::shallow_to_python = true;
+      if(Input())
+        {
+          stream = std::make_shared<std::stringstream>
+            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-1]));
+          *this & version_needed;
+          for(auto& libversion : version_needed)
+            if(libversion.second > GetLibraryVersion(libversion.first))
+              throw Exception("Error in unpickling data:\nLibrary " + libversion.first +
+                              " must be at least " + libversion.second.to_string());
+          stream = std::make_shared<std::stringstream>
+            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-2]));
+          *this & version_map;
+          stream = std::make_shared<std::stringstream>
+            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-3]));
+        }
+    }
+
+    void NeedsVersion(const std::string& library, const std::string& version) override
+    {
+      if(Output())
+        {
+          version_needed[library] = version_needed[library] > version ? version_needed[library] : version;
+        }
+    }
+
+    using ARCHIVE::Output;
+    using ARCHIVE::Input;
+    using ARCHIVE::FlushBuffer;
+    using ARCHIVE::operator&;
+    using ARCHIVE::operator<<;
+    using ARCHIVE::GetVersion;
+    void ShallowOutPython(const pybind11::object& val) override { lst.append(val); }
+    void ShallowInPython(pybind11::object& val) override { val = lst[index++]; }
+
+    pybind11::list WriteOut()
+    {
+      auto version_runtime = GetLibraryVersions();
+      FlushBuffer();
+      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
+      stream = std::make_shared<std::stringstream>();
+      *this & version_runtime;
+      FlushBuffer();
+      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
+      stream = std::make_shared<std::stringstream>();
+      *this & version_needed;
+      FlushBuffer();
+      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
+      return lst;
+    }
+  };
+
+  template<typename T, typename T_ARCHIVE_OUT=BinaryOutArchive, typename T_ARCHIVE_IN=BinaryInArchive>
+  auto NGSPickle()
+  {
+    return pybind11::pickle([](T* self)
+                      {
+                        PyArchive<T_ARCHIVE_OUT> ar;
+                        ar.SetParallel(parallel_pickling);
+                        ar & self;
+                        auto output = pybind11::make_tuple(ar.WriteOut());
+                        return output;
+                      },
+                      [](const pybind11::tuple & state)
+                      {
+                        T* val = nullptr;
+                        PyArchive<T_ARCHIVE_IN> ar(state[0]);
+                        ar & val;
+                        return val;
+                      });
+  }
+
   template<typename T>
   Array<T> makeCArray(const py::object& obj)
   {
@@ -80,19 +295,15 @@ namespace ngcore
     return arr;
   }
 
-  namespace detail
+  template <typename T>
+  // py::object makePyTuple (FlatArray<T> ar)
+  py::object makePyTuple (const BaseArrayObject<T> & ar)
   {
-    template<typename T>
-    struct HasPyFormat
-    {
-    private:
-      template<typename T2>
-      static auto check(T2*) -> std::enable_if_t<std::is_same_v<decltype(std::declval<py::format_descriptor<T2>>().format()), std::string>, std::true_type>;
-      static auto check(...) -> std::false_type;
-    public:
-      static constexpr bool value = decltype(check((T*) nullptr))::value;
-    };
-  } // namespace detail
+    py::tuple res(ar.Size());
+    for (auto i : Range(ar))
+      res[i] = py::cast(ar[i]);
+    return res;
+  }
 
   template <typename T, typename TIND=typename FlatArray<T>::index_type>
   void ExportArray (py::module &m)
@@ -108,8 +319,9 @@ namespace ngcore
         .def ("__getitem__",
               [](TFlat & self, TIND i) -> T&
                              {
-                               static constexpr int base = IndexBASE<TIND>();
-                               if (i < base || i >= self.Size()+base)
+                               // static constexpr int base = IndexBASE<TIND>();
+                               auto reli = i - IndexBASE<TIND>();
+                               if (reli < 0 || reli >= self.Size())
                                  throw py::index_error();
                                return self[i]; 
                              },
@@ -117,8 +329,9 @@ namespace ngcore
         .def ("__setitem__",
               [](TFlat & self, TIND i, T val) -> T&
                              {
-                               static constexpr int base = IndexBASE<TIND>();
-                               if (i < base || i >= self.Size()+base)
+                               // static constexpr int base = IndexBASE<TIND>();
+                               auto reli = i - IndexBASE<TIND>();                               
+                               if (reli < 0 || reli >= self.Size())
                                  throw py::index_error();
                                self[i] = val;
                                return self[i];
@@ -173,7 +386,7 @@ namespace ngcore
           }
 
       std::string aname = std::string("Array_") + suffix;
-      py::class_<TArray, TFlat>(m, aname.c_str())
+      auto arr = py::class_<TArray, TFlat> (m, aname.c_str())
         .def(py::init([] (size_t n) { return new TArray(n); }),py::arg("n"), "Makes array of given length")
         .def(py::init([] (std::vector<T> const & x)
                   {
@@ -183,10 +396,54 @@ namespace ngcore
                       tmp[TIND(i)] = x[i];
                     return tmp;
                   }), py::arg("vec"), "Makes array with given list of elements")
-
-      ;
+        ;
+      if constexpr(is_archivable<TArray>)
+        arr.def(NGSPickle<TArray>());
+      py::implicitly_convertible<std::vector<T>, TArray>();
     }
 
+  template <typename T>
+  void ExportTable (py::module &m)
+  {
+    py::class_<ngcore::Table<T>, std::shared_ptr<ngcore::Table<T>>> (m, ("Table_"+GetPyName<T>()).c_str())
+      .def(py::init([] (py::list blocks)
+                    {
+                       size_t size = py::len(blocks);
+                       Array<int> cnt(size);
+                       size_t i = 0;
+                       for (auto block : blocks)
+                         cnt[i++] = py::len(block);
+                       
+                       i = 0;
+                       Table<T> blocktable(cnt);
+                       for (auto block : blocks)
+                         {
+                           auto row = blocktable[i++];
+                           size_t j = 0;
+                           for (auto val : block)
+                             row[j++] = val.cast<T>();
+                         }
+                       // cout << "blocktable = " << *blocktable << endl;
+                       return blocktable;
+                      
+                    }), py::arg("blocks"), "a list of lists")
+
+      .def ("__len__", [] (Table<T> &self ) { return self.Size(); } )
+      .def ("__getitem__",
+            [](Table<T> & self, size_t i) -> FlatArray<T>
+            {
+              if (i >= self.Size())
+                throw py::index_error();
+              return self[i]; 
+            })
+      .def("__str__", [](Table<T> & self)
+           {
+             return ToString(self);
+           })
+      ;
+  }
+
+  
   void NGCORE_API SetFlag(Flags &flags, std::string s, py::object value);
   // Parse python kwargs to flags
   Flags NGCORE_API CreateFlagsFromKwArgs(const py::kwargs& kwargs, py::object pyclass = py::none(),
@@ -194,126 +451,6 @@ namespace ngcore
   // Create python dict from kwargs
   py::dict NGCORE_API CreateDictFromFlags(const Flags& flags);
 
-  // ***************  Archiving functionality  **************
-
-    template<typename T>
-    Archive& Archive :: Shallow(T& val)
-    {
-      static_assert(detail::is_any_pointer<T>, "ShallowArchive must be given pointer type!");
-#ifdef NETGEN_PYTHON
-      if(shallow_to_python)
-        {
-          if(is_output)
-            ShallowOutPython(pybind11::cast(val));
-          else
-          {
-            pybind11::object obj;
-            ShallowInPython(obj);
-            val = pybind11::cast<T>(obj);
-          }
-        }
-      else
-#endif // NETGEN_PYTHON
-        *this & val;
-      return *this;
-    }
-
-  template<typename ARCHIVE>
-  class NGCORE_API_EXPORT PyArchive : public ARCHIVE
-  {
-  private:
-    pybind11::list lst;
-    size_t index = 0;
-    std::map<std::string, VersionInfo> version_needed;
-  protected:
-    using ARCHIVE::stream;
-    using ARCHIVE::version_map;
-    using ARCHIVE::logger;
-    using ARCHIVE::GetLibraryVersions;
-  public:
-    PyArchive(const pybind11::object& alst = pybind11::none()) :
-      ARCHIVE(std::make_shared<std::stringstream>()),
-      lst(alst.is_none() ? pybind11::list() : pybind11::cast<pybind11::list>(alst))
-    {
-      ARCHIVE::shallow_to_python = true;
-      if(Input())
-        {
-          stream = std::make_shared<std::stringstream>
-            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-1]));
-          *this & version_needed;
-          logger->debug("versions needed for unpickling = {}", version_needed);
-          for(auto& libversion : version_needed)
-            if(libversion.second > GetLibraryVersion(libversion.first))
-              throw Exception("Error in unpickling data:\nLibrary " + libversion.first +
-                              " must be at least " + libversion.second.to_string());
-          stream = std::make_shared<std::stringstream>
-            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-2]));
-          *this & version_map;
-          stream = std::make_shared<std::stringstream>
-            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-3]));
-        }
-    }
-
-    void NeedsVersion(const std::string& library, const std::string& version) override
-    {
-      if(Output())
-        {
-          logger->debug("Need version {} of library {}.", version, library);
-          version_needed[library] = version_needed[library] > version ? version_needed[library] : version;
-        }
-    }
-
-    using ARCHIVE::Output;
-    using ARCHIVE::Input;
-    using ARCHIVE::FlushBuffer;
-    using ARCHIVE::operator&;
-    using ARCHIVE::operator<<;
-    using ARCHIVE::GetVersion;
-    void ShallowOutPython(const pybind11::object& val) override { lst.append(val); }
-    void ShallowInPython(pybind11::object& val) override { val = lst[index++]; }
-
-    pybind11::list WriteOut()
-    {
-      FlushBuffer();
-      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
-      stream = std::make_shared<std::stringstream>();
-      *this & GetLibraryVersions();
-      FlushBuffer();
-      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
-      stream = std::make_shared<std::stringstream>();
-      logger->debug("Writeout version needed = {}", version_needed);
-      *this & version_needed;
-      FlushBuffer();
-      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
-      return lst;
-    }
-  };
-
-  template<typename T, typename T_ARCHIVE_OUT=BinaryOutArchive, typename T_ARCHIVE_IN=BinaryInArchive>
-  auto NGSPickle()
-  {
-    return pybind11::pickle([](T* self)
-                      {
-                        PyArchive<T_ARCHIVE_OUT> ar;
-                        ar & self;
-                        auto output = pybind11::make_tuple(ar.WriteOut());
-                        GetLogger("Archive")->trace("Pickling output for object of type {} = {}",
-                                                    Demangle(typeid(T).name()),
-                                                    std::string(pybind11::str(output)));
-                        return output;
-                      },
-                      [](const pybind11::tuple & state)
-                      {
-                        T* val = nullptr;
-                        GetLogger("Archive")->trace("State for unpickling of object of type {} = {}",
-                                                    Demangle(typeid(T).name()),
-                                                    std::string(pybind11::str(state[0])));
-                        PyArchive<T_ARCHIVE_IN> ar(state[0]);
-                        ar & val;
-                        return val;
-                      });
-  }
-
 
 } // namespace ngcore
 

libsrc/core/python_ngcore_export.cpp

@@ -1,11 +1,18 @@
 #include "python_ngcore.hpp"
 #include "bitarray.hpp"
 #include "taskmanager.hpp"
+#include "mpi_wrapper.hpp"
 
 using namespace ngcore;
 using namespace std;
 using namespace pybind11::literals;
 
+namespace pybind11 { namespace detail {
+}} // namespace pybind11::detail
+
+
+
+
 PYBIND11_MODULE(pyngcore, m) // NOLINT
 {
   try
@@ -18,6 +25,23 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
   ExportArray<unsigned>(m);
   ExportArray<size_t>(m);
   ExportArray<double>(m);
+  ExportArray<float>(m);
+  ExportArray<signed short>(m);
+  ExportArray<signed char>(m);
+  ExportArray<unsigned short>(m);
+  ExportArray<unsigned char>(m);
+
+  // Compiler dependent implementation of size_t
+  if constexpr(!is_same_v<size_t, uint64_t>)
+    ExportArray<uint64_t>(m);
+
+  ExportTable<int>(m);
+
+  #ifdef PARALLEL
+  py::class_<NG_MPI_Comm> (m, "_NG_MPI_Comm")
+          ;
+  m.def("InitMPI", &InitMPI, py::arg("mpi_library_path")=nullopt);
+  #endif // PARALLEL
 
   py::class_<BitArray, shared_ptr<BitArray>> (m, "BitArray")
     .def(py::init([] (size_t n) { return make_shared<BitArray>(n); }),py::arg("n"))
@@ -35,12 +59,14 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
     .def("__len__", &BitArray::Size)
     .def("__getitem__", [] (BitArray & self, int i)
                                          {
+                                           if (i < 0) i+=self.Size();
                                            if (i < 0 || i >= self.Size())
                                              throw py::index_error();
                                            return self.Test(i);
                                          }, py::arg("pos"), "Returns bit from given position")
     .def("__setitem__", [] (BitArray & self, int i, bool b)
                                          {
+                                           if (i < 0) i+=self.Size();
                                            if (i < 0 || i >= self.Size())
                                              throw py::index_error();
                                            if (b) self.SetBit(i); else self.Clear(i);
@@ -115,20 +141,34 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
 
     .def(py::self | py::self)
     .def(py::self & py::self)
+    #ifdef __clang__ 
+    // see https://github.com/pybind/pybind11/issues/1893
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wself-assign-overloaded"
+    #endif
     .def(py::self |= py::self)
     .def(py::self &= py::self)
+    #ifdef __clang__
+    #pragma GCC diagnostic pop
+    #endif
     .def(~py::self)
     ;
 
   py::class_<Flags>(m, "Flags")
     .def(py::init<>())
     .def("__str__", &ToString<Flags>)
-    .def(py::init([](py::object & obj) {
+    .def(py::init([](py::dict kwargs) {
           Flags flags;
-          py::dict d(obj);          
-          SetFlag (flags, "", d);
+          for (auto d : kwargs)
+            SetFlag(flags, d.first.cast<string>(), d.second.cast<py::object>());
           return flags;
-        }), py::arg("obj"), "Create Flags by given object")
+    }), "Create flags from dict")
+    .def(py::init([](py::kwargs kwargs) {
+          Flags flags;
+          for (auto d : kwargs)
+            SetFlag(flags, d.first.cast<string>(), d.second.cast<py::object>());
+          return flags;
+        }), "Create flags from kwargs")
     .def(py::pickle([] (const Flags& self)
         {
           std::stringstream str;
@@ -156,6 +196,9 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
         return self;
     }, py::arg("akey"), py::arg("value"), "Set flag by given value.")
 
+    .def("keys", [](Flags & self) -> py::list {
+        return CreateDictFromFlags(self).attr("keys")();
+    })
     .def("__getitem__", [](Flags & self, const string& name) -> py::object {
 
 	  if(self.NumListFlagDefined(name))
@@ -175,6 +218,14 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
 
 	  return py::cast(self.GetDefineFlag(name));
       }, py::arg("name"), "Return flag by given name")
+    .def("ToDict", [](const Flags& flags)
+    {
+      return CreateDictFromFlags(flags);
+    })
+    .def("items", [](const Flags& flags)
+    {
+      return CreateDictFromFlags(flags).attr("items")();
+    })
   ;
   py::implicitly_convertible<py::dict, Flags>();
 
@@ -227,7 +278,10 @@ threads : int
   class ParallelContextManager {
       int num_threads;
     public:
-      ParallelContextManager() : num_threads(0) {};
+      ParallelContextManager() : num_threads(0) {
+        TaskManager::SetPajeTrace(0);
+        PajeTrace::SetMaxTracefileSize(0);
+      };
       ParallelContextManager(size_t pajesize) : num_threads(0) {
         TaskManager::SetPajeTrace(pajesize > 0);
         PajeTrace::SetMaxTracefileSize(pajesize);
@@ -246,4 +300,94 @@ threads : int
     .def("__timing__", &TaskManager::Timing)
     ;
 
+  py::class_<PajeTrace>(m, "PajeTrace")
+    .def(py::init( [] (string filename, size_t size_mb, bool threads, bool thread_counter, bool memory)
+          {
+              PajeTrace::SetMaxTracefileSize(size_mb*1014*1024);
+              PajeTrace::SetTraceThreads(threads);
+              PajeTrace::SetTraceMemory(memory);
+              PajeTrace::SetTraceThreadCounter(thread_counter);
+              trace = new PajeTrace(TaskManager::GetMaxThreads(), filename);
+              return trace;
+          }), py::arg("filename")="ng.trace", py::arg("size")=1000,
+              py::arg("threads")=true, py::arg("thread_counter")=false,
+              py::arg("memory")=true,
+              "size in Megabytes"
+        )
+    .def("__enter__", [](PajeTrace & self) { })
+    .def("__exit__", [](PajeTrace & self, py::args) { trace = nullptr; })
+    .def_static("SetTraceThreads", &PajeTrace::SetTraceThreads)
+    .def_static("SetTraceThreadCounter", &PajeTrace::SetTraceThreadCounter)
+    .def_static("SetMaxTracefileSize", &PajeTrace::SetMaxTracefileSize)
+#ifdef NETGEN_TRACE_MEMORY
+    .def_static("WriteMemoryChart", [](string filename){ if(trace) trace->WriteMemoryChart(filename); }, py::arg("filename")="memory" )
+#endif // NETGEN_TRACE_MEMORY
+    ;
+
+    m.def("GetTotalMemory", MemoryTracer::GetTotalMemory);
+
+    py::class_<Timer<>> (m, "Timer")
+    .def(py::init<const string&>())
+    .def("Start", static_cast<void (Timer<>::*)()const>(&Timer<>::Start), "start timer")
+    .def("Stop", static_cast<void (Timer<>::*)()const>(&Timer<>::Stop), "stop timer")
+    .def_property_readonly("time", &Timer<>::GetTime, "returns time")
+    .def("__enter__", static_cast<void (Timer<>::*)()const>(&Timer<>::Start))
+    .def("__exit__", [](Timer<>& t, py::object, py::object, py::object)
+                     {
+                       t.Stop();
+                     })
+    ;
+  
+  m.def("Timers",
+	  []() 
+	   {
+	     py::list timers;
+	     for (int i = 0; i < NgProfiler::SIZE; i++)
+	       if (!NgProfiler::timers[i].name.empty())
+               {
+                 py::dict timer;
+                 timer["name"] = py::str(NgProfiler::timers[i].name);
+                 timer["time"] = py::float_(NgProfiler::GetTime(i));
+                 timer["counts"] = py::int_(NgProfiler::GetCounts(i));
+                 timer["flops"] = py::float_(NgProfiler::GetFlops(i));
+                 timer["Gflop/s"] = py::float_(NgProfiler::GetFlops(i)/NgProfiler::GetTime(i)*1e-9);
+                 timers.append(timer);
+               }
+	     return timers;
+	   }, "Returns list of timers"
+	   );
+  m.def("ResetTimers", &NgProfiler::Reset);
+
+  py::class_<NgMPI_Comm> (m, "MPI_Comm")
+#ifdef PARALLEL
+    .def(py::init([] (mpi4py_comm comm) { return NgMPI_Comm(comm); }))
+    .def("WTime", [](NgMPI_Comm  & c) { return NG_MPI_Wtime(); })
+    .def_property_readonly ("mpi4py", [](NgMPI_Comm & self) { return NG_MPI_CommToMPI4Py(self); })
+#endif  // PARALLEL
+    .def_property_readonly ("rank", &NgMPI_Comm::Rank)
+    .def_property_readonly ("size", &NgMPI_Comm::Size)
+    .def("Barrier", &NgMPI_Comm::Barrier)
+    .def("Sum", [](NgMPI_Comm  & c, double x) { return c.AllReduce(x, NG_MPI_SUM); })
+    .def("Min", [](NgMPI_Comm  & c, double x) { return c.AllReduce(x, NG_MPI_MIN); })
+    .def("Max", [](NgMPI_Comm  & c, double x) { return c.AllReduce(x, NG_MPI_MAX); })
+    .def("Sum", [](NgMPI_Comm  & c, int x) { return c.AllReduce(x, NG_MPI_SUM); })
+    .def("Min", [](NgMPI_Comm  & c, int x) { return c.AllReduce(x, NG_MPI_MIN); })
+    .def("Max", [](NgMPI_Comm  & c, int x) { return c.AllReduce(x, NG_MPI_MAX); })
+    .def("Sum", [](NgMPI_Comm  & c, size_t x) { return c.AllReduce(x, NG_MPI_SUM); })
+    .def("Min", [](NgMPI_Comm  & c, size_t x) { return c.AllReduce(x, NG_MPI_MIN); })
+    .def("Max", [](NgMPI_Comm  & c, size_t x) { return c.AllReduce(x, NG_MPI_MAX); })
+    .def("SubComm", [](NgMPI_Comm & c, std::vector<int> proc_list) {
+        Array<int> procs(proc_list.size());
+        for (int i = 0; i < procs.Size(); i++)
+          { procs[i] = proc_list[i]; }
+        if (!procs.Contains(c.Rank()))
+          { throw Exception("rank "+ToString(c.Rank())+" not in subcomm"); }
+	return c.SubCommunicator(procs);
+      }, py::arg("procs"));
+  ;
+
+    
+#ifdef PARALLEL
+  py::implicitly_convertible<mpi4py_comm, NgMPI_Comm>();
+#endif // PARALLEL
 }

libsrc/core/ranges.hpp

@@ -0,0 +1,109 @@
+#ifndef NETGEN_CORE_RANGES_HPP
+#define NETGEN_CORE_RANGES_HPP
+
+#include <iterator>
+
+namespace ngcore
+{
+  template<typename Iterator>
+  class AdapterRange
+  {
+    Iterator _begin,_end;
+  public:
+    AdapterRange(Iterator abegin, Iterator aend) : _begin(abegin), _end(aend) { ; }
+    Iterator begin() const { return _begin; }
+    Iterator end() const { return _end; }
+  };
+
+  template<typename FUNC>
+  class FilterAdapter
+  {
+    FUNC f;
+  public:
+    FilterAdapter(FUNC af) : f(af) { ; }
+    FUNC GetFunction() const { return f; }
+  };
+
+  template<typename FUNC, typename Iterator>
+  class FilterIterator
+  {
+    Iterator iter;
+    Iterator end;
+    FUNC f;
+  public:
+    FilterIterator(FUNC af, Iterator aiter, Iterator aend)
+      :  iter(aiter), end(aend), f(af)
+    {
+      while(iter!=end && !f(*iter))
+        ++iter;
+    }
+    inline FilterIterator& operator ++()
+    {
+      ++iter;
+      while(iter!=end && !f(*iter))
+        ++iter;
+      return *this;
+    }
+
+    inline bool operator !=(FilterIterator other)
+    {
+      return iter != other.iter;
+    }
+
+    inline bool operator ==(FilterIterator other)
+    {
+      return iter == other.iter;
+    }
+
+    inline decltype(auto) operator *() const
+    {
+      return *iter;
+    }
+  };
+
+  template<typename FUNC>
+  FilterAdapter<FUNC> filter(FUNC f) { return {f}; }
+
+  template<typename Range, typename FUNC>
+  auto operator |(Range&& range, FilterAdapter<FUNC> adapter)
+    -> AdapterRange<FilterIterator<FUNC,decltype(std::begin(range))>>
+  {
+    return {{adapter.GetFunction(),std::begin(range),std::end(range)},
+        {adapter.GetFunction(), std::end(range), std::end(range)}};
+  }
+
+  template<typename FUNC, typename Iterator>
+  class TransformIterator
+  {
+    FUNC f;
+    Iterator iter;
+  public:
+    TransformIterator(FUNC af, Iterator aiter) : f(af), iter(aiter) { ; }
+
+    TransformIterator& operator++() { ++iter; }
+    bool operator !=(TransformIterator other) { return iter != other.iter; }
+    decltype(auto) operator *() const { return f(*iter); }
+  };
+
+  template<typename FUNC>
+  class TransformAdapter
+  {
+    FUNC f;
+  public:
+    TransformAdapter(FUNC af) : f(af) { ; }
+    FUNC GetFunction() const { return f; }
+  };
+
+  template<typename FUNC>
+  TransformAdapter<FUNC> transform(FUNC f) { return {f}; }
+
+  template<typename Range, typename FUNC>
+  auto operator |(Range&& range, TransformAdapter<FUNC> adapter)
+    -> AdapterRange<TransformIterator<FUNC,decltype(std::begin(range))>>
+  {
+    return {{adapter.GetFunction(), std::begin(range)},
+        {adapter.GetFunction(),std::end(range)}};
+  }
+} // namespace ngcore
+
+#endif // NETGEN_CORE_RANGES_HPP

libsrc/core/register_archive.hpp

@@ -0,0 +1,92 @@
+#ifndef NETGEN_REGISTER_ARCHIVE_HPP
+#define NETGEN_REGISTER_ARCHIVE_HPP
+
+#ifdef NETGEN_PYTHON
+#include <memory>
+#include <pybind11/pybind11.h>
+#include <pybind11/cast.h>
+#endif // NETGEN_PYTHON
+#include <tuple>
+
+#include "archive.hpp"
+
+namespace ngcore {
+  // ***************  Archiving functionality  **************
+
+#ifdef NETGEN_PYTHON
+  template<typename T>
+  Archive& Archive :: Shallow(T& val)
+  {
+    static_assert(detail::is_any_pointer<T>, "ShallowArchive must be given pointer type!");
+    if(shallow_to_python)
+      {
+        if(is_output)
+          ShallowOutPython(pybind11::cast(val));
+        else
+        {
+          pybind11::object obj;
+          ShallowInPython(obj);
+          val = pybind11::cast<T>(obj);
+        }
+      }
+    else
+      *this & val;
+    return *this;
+  }
+
+  /*
+    // now using has_shared_from_this2 in archive.hpp
+  template <typename T>
+  struct has_shared_from_this
+  {
+    template <typename C> static std::true_type check( decltype( sizeof(&C::shared_from_this )) ) { return std::true_type(); }
+    template <typename> static std::false_type check(...) { return std::false_type(); }
+    typedef decltype( check<T>(sizeof(char)) ) type;
+    static constexpr type value = type();
+  };
+  */
+#endif // NETGEN_PYTHON
+
+
+  template<typename T, typename Bases=std::tuple<>>
+  class RegisterClassForArchive
+  {
+  public:
+    RegisterClassForArchive()
+    {
+      static_assert(std::is_base_of<Bases, T>::value ||
+                    detail::is_base_of_tuple<T, Bases>,
+                    "Second argument must be base class or tuple of base classes of T");
+      detail::ClassArchiveInfo info {};
+      info.creator = [](const std::type_info& ti, Archive& ar) -> void*
+      {
+        detail::TCargs<T> args;
+        ar &args;
+        auto nT = detail::constructIfPossible<T>(std::move(args));
+        return typeid(T) == ti ? nT
+          : Archive::Caster<T, Bases>::tryUpcast(ti, nT);
+      };
+      info.upcaster = [](const std::type_info& ti, void* p) -> void*
+      { return typeid(T) == ti ? p : Archive::Caster<T, Bases>::tryUpcast(ti, static_cast<T*>(p)); };
+      info.downcaster = [](const std::type_info& ti, void* p) -> void*
+      { return typeid(T) == ti ? p : Archive::Caster<T, Bases>::tryDowncast(ti, p); };
+      info.cargs_archiver = [](Archive &ar, void* p) {
+        if constexpr(detail::has_GetCArgs_v<T>)
+          ar << static_cast<T*>(p)->GetCArgs();
+      };
+#ifdef NETGEN_PYTHON
+    info.anyToPyCaster = [](const std::any &a) {
+      if constexpr(has_shared_from_this2<T>::value) {
+        std::shared_ptr<T> val = std::any_cast<std::shared_ptr<T>>(a);
+        return pybind11::cast(val);
+      } else {
+        const T* val = std::any_cast<T>(&a);
+        return pybind11::cast(val);
+      }
+    };
+#endif // NETGEN_PYTHON
+    Archive::SetArchiveRegister(std::string(Demangle(typeid(T).name())),info);
+  }
+};
+} // namespace ngcore
+#endif // NETGEN_REGISTER_ARCHIVE_HPP

libsrc/core/signal.hpp

@@ -2,6 +2,7 @@
 #define NGCORE_SIGNALS_HPP
 
 #include <list>
+#include <map>
 #include <functional>
 
 namespace ngcore
@@ -43,6 +44,39 @@ namespace ngcore
     }
     inline bool GetEmitting() const { return is_emitting; }
   };
+
+
+
+
+  class SimpleSignal
+  {
+  private:
+    // std::map<void*,std::function<void()>> funcs;
+    std::list<std::pair<void*,std::function<void()>>> funcs;
+  public:
+    SimpleSignal() = default;
+
+    template<typename FUNC>
+    void Connect(void* var, FUNC f)
+    {
+      // funcs[var] = f;
+      funcs.push_back ( { var, f } );
+    }
+
+    void Remove(void* var)
+    {
+      // funcs.erase(var);
+      funcs.remove_if([&] (auto var_f) { return var_f.first==var; });
+    }
+
+    inline void Emit()
+    {
+      for (auto [key,f] : funcs)
+        f();
+    }
+  };
+
+  
 } // namespace ngcore
 
 #endif // NGCORE_SIGNALS_HPP

libsrc/core/simd.hpp

@@ -0,0 +1,92 @@
+#ifndef NETGEN_CORE_SIMD_HPP
+#define NETGEN_CORE_SIMD_HPP
+
+/**************************************************************************/
+/* File:   simd.hpp                                                       */
+/* Author: Joachim Schoeberl, Matthias Hochsteger                         */
+/* Date:   25. Mar. 16                                                    */
+/**************************************************************************/
+
+#include "ngcore_api.hpp"
+
+#include "simd_generic.hpp"
+
+#ifndef __CUDA_ARCH__
+
+#ifdef NETGEN_ARCH_AMD64
+#ifndef __SSE__
+#define __SSE__
+#endif
+#include "simd_sse.hpp"
+#endif
+
+#ifdef __AVX__
+#include "simd_avx.hpp"
+#endif
+
+#ifdef __AVX512F__
+#include "simd_avx512.hpp"
+#endif
+
+#ifdef __aarch64__
+#include "simd_arm64.hpp"
+#endif
+
+#endif // __CUDA_ARCH__
+
+namespace ngcore
+{
+#ifndef __CUDA_ARCH__
+#ifdef NETGEN_ARCH_AMD64
+  /*
+  NETGEN_INLINE auto HSum (SIMD<double,2> v1, SIMD<double,2> v2, SIMD<double,2> v3, SIMD<double,2> v4)
+  {
+    SIMD<double,2> hsum1 = my_mm_hadd_pd (v1.Data(), v2.Data());
+    SIMD<double,2> hsum2 = my_mm_hadd_pd (v3.Data(), v4.Data());
+    return SIMD<double,4> (hsum1, hsum2);
+  }
+  */
+  
+  NETGEN_INLINE auto GetMaskFromBits( unsigned int i )
+  {
+    return SIMD<mask64>::GetMaskFromBits(i);
+  }
+#endif
+#endif // __CUDA_ARCH__
+  
+  NETGEN_INLINE void SIMDTranspose (SIMD<double,4> a1, SIMD<double,4> a2, SIMD <double,4> a3, SIMD<double,4> a4,
+                                    SIMD<double,4> & b1, SIMD<double,4> & b2, SIMD<double,4> & b3, SIMD<double,4> & b4)
+  {
+    if constexpr (sizeof(a1.Lo()) == 16)
+      {
+        auto [h1,h2] = Unpack(a1,a2);
+        auto [h3,h4] = Unpack(a3,a4);
+        b1 = SIMD<double,4> (h1.Lo(), h3.Lo());
+        b2 = SIMD<double,4> (h2.Lo(), h4.Lo());
+        b3 = SIMD<double,4> (h1.Hi(), h3.Hi());
+        b4 = SIMD<double,4> (h2.Hi(), h4.Hi());
+      }
+    else
+      {
+        b1 = SIMD<double,4> (a1[0], a2[0], a3[0], a4[0]);
+        b2 = SIMD<double,4> (a1[1], a2[1], a3[1], a4[1]);
+        b3 = SIMD<double,4> (a1[2], a2[2], a3[2], a4[2]);
+        b4 = SIMD<double,4> (a1[3], a2[3], a3[3], a4[3]);
+      }
+  }
+  
+  template<int N>
+  NETGEN_INLINE auto HSum (SIMD<double,N> s1, SIMD<double,N> s2)
+  {
+    return SIMD<double,2>(HSum(s1), HSum(s2));
+  }
+
+  template<int N>
+  NETGEN_INLINE auto HSum (SIMD<double,N> s1, SIMD<double,N> s2, SIMD<double,N> s3, SIMD<double,N> s4 )
+  {
+    // return SIMD<double,4>(HSum(s1), HSum(s2), HSum(s3), HSum(s4));
+    return SIMD<double,4>(HSum(s1, s2), HSum(s3,s4));
+  }
+}
+
+#endif // NETGEN_CORE_SIMD_HPP

libsrc/core/simd_arm64.hpp

@@ -0,0 +1,203 @@
+#include "arm_neon.h"
+
+namespace ngcore
+{
+
+  template <>
+  class SIMD<mask64,2>
+  {
+    int64x2_t mask;
+  public:
+    SIMD (int i)
+    {
+      mask[0] = i > 0 ? -1 : 0;
+      mask[1] = i > 1 ? -1 : 0;
+    }
+
+    SIMD (bool i0, bool i1) { mask[0] = i0 ? -1 : 0; mask[1] = i1 ? -1 : 0; }
+    SIMD (SIMD<mask64,1> i0, SIMD<mask64,1> i1) { mask[0] = i0[0]; mask[1] = i1[0]; }
+    // SIMD (float64x2_t _data) : mask{_data} { }
+    SIMD (int64x2_t _data) : mask{_data} { }
+    auto Data() const { return mask; }
+    static constexpr int Size() { return 2; }
+    // static NETGEN_INLINE SIMD<mask64, 2> GetMaskFromBits (unsigned int i);
+    int64_t operator[] (int i) const { return mask[i]; }
+
+    template <int I>
+    int64_t Get() const { return mask[I]; }
+    
+    auto Lo() const { return mask[0]; }
+    auto Hi() const { return mask[1]; }
+  };
+
+  
+  template<>
+  class SIMD<double,2>
+  {
+    float64x2_t data;
+
+  public:
+    static constexpr int Size() { return 2; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    // SIMD (double v0, double v1) : data{v0,v1} { }
+    SIMD (double v0, double v1) : data{vcombine_f64(float64x1_t{v0}, float64x1_t{v1})} { }
+    SIMD (SIMD<double,1> v0, SIMD<double,1> v1) : data{vcombine_f64(float64x1_t{v0.Data()}, float64x1_t{v1.Data()})} { }
+    SIMD (std::array<double, 2> arr) : data{arr[0], arr[1]} { } 
+
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (double val) : data{val,val} { }
+    SIMD (int val)    : data{double(val),double(val)} { }
+    SIMD (size_t val) : data{double(val),double(val)} { }
+
+    SIMD (double const * p)
+    {
+      data = vld1q_f64(p);      
+      // data[0] = p[0];
+      // data[1] = p[1];
+    }
+    
+    SIMD (double const * p, SIMD<mask64,2> mask)
+      {
+	data[0] = mask[0] ? p[0] : 0;
+	data[1] = mask[1] ? p[1] : 0;
+      }
+    SIMD (float64x2_t _data) { data = _data; }
+    
+    template<typename T, typename std::enable_if<std::is_convertible<T, std::function<double(int)>>::value, int>::type = 0>
+    SIMD (const T & func)
+    {
+      data[0] = func(0);
+      data[1] = func(1);
+    }
+
+    void Store (double * p)
+    {
+      vst1q_f64(p, data);
+      /*
+      p[0] = data[0];
+      p[1] = data[1];
+      */
+    }
+    
+    void Store (double * p, SIMD<mask64,2> mask)
+    {
+      if (mask[0]) p[0] = data[0];
+      if (mask[1]) p[1] = data[1];
+    }
+    
+    // NETGEN_INLINE double operator[] (int i) const { return ((double*)(&data))[i]; }
+    NETGEN_INLINE double operator[] (int i) const { return data[i]; }
+    NETGEN_INLINE double & operator[] (int i)  { return ((double*)&data)[i]; }
+
+    template <int I>
+    double Get() const { return data[I]; }
+    
+    NETGEN_INLINE auto Data() const { return data; }
+    NETGEN_INLINE auto & Data() { return data; }
+
+    operator std::tuple<double&,double&> ()
+    {
+      auto pdata = (double*)&data;
+      return std::tuple<double&,double&>(pdata[0], pdata[1]);
+    }
+    
+    double Lo() const { return Get<0>(); } // data[0]; }
+    double Hi() const { return Get<1>(); } // data[1]; }
+    // double Hi() const { return vget_high_f64(data)[0]; }
+  };
+
+
+
+  NETGEN_INLINE double HSum (SIMD<double,2> sd)
+  {
+    return sd.Lo()+sd.Hi();  // sd[0]+sd[1];
+  }
+
+  NETGEN_INLINE SIMD<double,2> HSum (SIMD<double,2> a, SIMD<double,2> b)
+  {
+    // return SIMD<double,2> (a[0]+a[1], b[0]+b[1]);
+    return vpaddq_f64(a.Data(), b.Data());
+  }
+
+  NETGEN_INLINE SIMD<double,4> HSum(SIMD<double,2> a, SIMD<double,2> b, SIMD<double,2> c, SIMD<double,2> d)
+  {
+    return SIMD<double,4> (HSum(a,b), HSum(c,d));
+  }
+
+
+  NETGEN_INLINE SIMD<double,2>  SwapPairs (SIMD<double,2> a)
+  {
+    return __builtin_shufflevector(a.Data(), a.Data(), 1, 0);
+  }
+  
+
+
+  // a*b+c
+  NETGEN_INLINE SIMD<double,2> FMA (SIMD<double,2> a, SIMD<double,2> b, SIMD<double,2> c)
+  {
+    return vmlaq_f64(c.Data(), a.Data(), b.Data());
+  }
+  NETGEN_INLINE SIMD<double,2> FMA (const double & a, SIMD<double,2> b, SIMD<double,2> c)
+  {
+    return FMA(SIMD<double,2> (a), b, c);
+  }
+  // -a*b+c
+  NETGEN_INLINE SIMD<double,2> FNMA (SIMD<double,2> a, SIMD<double,2> b, SIMD<double,2> c)
+  {
+    return vmlsq_f64(c.Data(), a.Data(), b.Data());    
+    // return c-a*b;
+  }
+  NETGEN_INLINE SIMD<double,2> FNMA (const double & a, SIMD<double,2> b, SIMD<double,2> c)
+  {
+    return FNMA(SIMD<double,2> (a), b, c);
+  }
+
+  // ARM complex mult:
+  // https://arxiv.org/pdf/1901.07294.pdf
+  // c += a*b    (a0re, a0im, a1re, a1im, ...), 
+  NETGEN_INLINE void FMAComplex (SIMD<double,2> a, SIMD<double,2> b, SIMD<double,2> & c)
+  {
+    auto tmp = vcmlaq_f64(c.Data(), a.Data(), b.Data());   // are * b
+    c = vcmlaq_rot90_f64(tmp, a.Data(), b.Data());    // += i*aim * b
+  }
+  
+
+  NETGEN_INLINE SIMD<double,2> operator+ (SIMD<double,2> a, SIMD<double,2> b)
+  { return a.Data()+b.Data(); }
+  
+  NETGEN_INLINE SIMD<double,2> operator- (SIMD<double,2> a, SIMD<double,2> b)
+  { return a.Data()-b.Data(); }
+  NETGEN_INLINE SIMD<double,2> operator- (SIMD<double,2> a)
+  { return -a.Data(); }
+  
+  NETGEN_INLINE SIMD<double,2> operator* (SIMD<double,2> a, SIMD<double,2> b)
+  { return a.Data()*b.Data(); }
+  
+  NETGEN_INLINE SIMD<double,2> operator/ (SIMD<double,2> a, SIMD<double,2> b)
+  { return a.Data()/b.Data(); }
+
+
+  
+  NETGEN_INLINE SIMD<double,2> If (SIMD<mask64,2> a, SIMD<double,2> b, SIMD<double,2> c)
+  {
+    // return { a[0] ? b[0] : c[0], a[1] ? b[1] : c[1] };
+    uint64x2_t mask = vreinterpretq_u64_s64(a.Data());
+    return vbslq_f64(mask, b.Data(), c.Data());
+  }
+  NETGEN_INLINE SIMD<int64_t,2> If (SIMD<mask64,2> a, SIMD<int64_t,2> b, SIMD<int64_t,2> c)
+  {
+    return SIMD<int64_t,2> (a[0] ? b[0] : c[0], a[1] ? b[1] : c[1]);
+  }
+
+  NETGEN_INLINE SIMD<mask64,2> operator&& (SIMD<mask64,2> a, SIMD<mask64,2> b)
+  {
+    uint64x2_t m1 = vreinterpretq_u64_s64(a.Data());
+    uint64x2_t m2 = vreinterpretq_u64_s64(b.Data());
+    uint64x2_t res = vandq_u64 (m1, m2);
+    return vreinterpretq_s64_u64(res);
+  }
+  
+}
+

libsrc/core/simd_avx.hpp

@@ -0,0 +1,326 @@
+#ifndef NETGEN_CORE_SIMD_AVX_HPP
+#define NETGEN_CORE_SIMD_AVX_HPP
+
+/**************************************************************************/
+/* File:   simd_avx.hpp                                                   */
+/* Author: Joachim Schoeberl, Matthias Hochsteger                         */
+/* Date:   25. Mar. 16                                                    */
+/**************************************************************************/
+
+#include <immintrin.h>
+
+namespace ngcore
+{
+
+#if defined(__GNUC__) && (__GNUC__ == 7)
+  // GCC7 does not have intrinsic _mm256_set_m128i, see
+  // https://stackoverflow.com/questions/32630458/setting-m256i-to-the-value-of-two-m128i-values
+  NETGEN_INLINE auto _mm256_set_m128i(__m128i v0, __m128i v1) {
+      return _mm256_insertf128_si256(_mm256_castsi128_si256(v1), (v0), 1);
+  }
+#endif // defined(__GNUC__) && (__GNUC__ == 7)
+
+#if defined(__AVX2__)
+  NETGEN_INLINE __m256i my_mm256_cmpgt_epi64 (__m256i a, __m256i b)
+  {
+    return _mm256_cmpgt_epi64 (a,b);
+  }
+#else
+  NETGEN_INLINE __m256i my_mm256_cmpgt_epi64 (__m256i a, __m256i b)
+  {
+    __m128i rlo = _mm_cmpgt_epi64(_mm256_extractf128_si256(a, 0),
+                                  _mm256_extractf128_si256(b, 0));
+    __m128i rhi = _mm_cmpgt_epi64(_mm256_extractf128_si256(a, 1),
+                                  _mm256_extractf128_si256(b, 1));
+    return _mm256_insertf128_si256 (_mm256_castsi128_si256(rlo), rhi, 1);
+  }
+#endif
+
+
+  template <>
+  class SIMD<mask64,4>
+  {
+    __m256i mask;
+  public:
+    SIMD (int64_t i)
+      : mask(my_mm256_cmpgt_epi64(_mm256_set1_epi64x(i),
+                                  _mm256_set_epi64x(3, 2, 1, 0)))
+    { ; }
+    SIMD (__m256i _mask) : mask(_mask) { ; }
+    SIMD (__m256d _mask) : mask(_mm256_castpd_si256(_mask)) { ; }
+    __m256i Data() const { return mask; }
+    static constexpr int Size() { return 4; }
+    static SIMD<mask64, 4> GetMaskFromBits (unsigned int i);
+  };
+
+  static SIMD<mask64, 4> masks_from_4bits[16] = {
+    _mm256_set_epi64x (0,0,0,0), _mm256_set_epi64x (0,0,0,-1),
+    _mm256_set_epi64x (0,0,-1,0), _mm256_set_epi64x (0,0,-1,-1),
+    _mm256_set_epi64x (0,-1,0,0), _mm256_set_epi64x (0,-1,0,-1),
+    _mm256_set_epi64x (0,-1,-1,0), _mm256_set_epi64x (0,-1,-1,-1),
+    _mm256_set_epi64x (-1,0,0,0), _mm256_set_epi64x (-1,0,0,-1),
+    _mm256_set_epi64x (-1,0,-1,0), _mm256_set_epi64x (-1,0,-1,-1),
+    _mm256_set_epi64x (-1,-1,0,0), _mm256_set_epi64x (-1,-1,0,-1),
+    _mm256_set_epi64x (-1,-1,-1,0), _mm256_set_epi64x (-1,-1,-1,-1)
+  };
+
+  NETGEN_INLINE SIMD<mask64, 4> SIMD<mask64, 4> :: GetMaskFromBits (unsigned int i)
+  {
+    return masks_from_4bits[i & 15];
+  }
+
+  template<>
+  class alignas(32) SIMD<int64_t,4>
+  {
+    __m256i data;
+
+  public:
+    static constexpr int Size() { return 4; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (int64_t val) { data = _mm256_set1_epi64x(val); }
+    SIMD (int64_t v0, int64_t v1, int64_t v2, int64_t v3) { data = _mm256_set_epi64x(v3,v2,v1,v0); }
+    SIMD (std::array<int64_t,4> a)
+      : data{_mm256_set_epi64x(a[3],a[2],a[1],a[0])}
+    {}
+    SIMD (SIMD<int64_t,2> v0, SIMD<int64_t,2> v1)
+        : data(_mm256_set_m128i(v0.Data(),v1.Data()))
+      {}
+    SIMD (__m256i _data) { data = _data; }
+
+    NETGEN_INLINE auto operator[] (int i) const { return ((int64_t*)(&data))[i]; }
+    NETGEN_INLINE __m256i Data() const { return data; }
+    NETGEN_INLINE __m256i & Data() { return data; }
+
+    SIMD<int64_t,2> Lo() const { return _mm256_extractf128_si256(data, 0); }
+    SIMD<int64_t,2> Hi() const { return _mm256_extractf128_si256(data, 1); }
+    static SIMD FirstInt(int n0=0) { return { n0+0, n0+1, n0+2, n0+3 }; }
+  };
+
+
+  NETGEN_INLINE SIMD<int64_t,4> operator-(SIMD<int64_t,4> a) { return _mm256_sub_epi64(_mm256_setzero_si256(), a.Data()); }
+
+#ifdef __AVX2__
+  NETGEN_INLINE SIMD<int64_t,4> operator+ (SIMD<int64_t,4> a, SIMD<int64_t,4> b) { return _mm256_add_epi64(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<int64_t,4> operator- (SIMD<int64_t,4> a, SIMD<int64_t,4> b) { return _mm256_sub_epi64(a.Data(),b.Data()); }
+#endif // __AVX2__
+
+  template<>
+  class alignas(32) SIMD<double,4>
+  {
+    __m256d data;
+
+  public:
+    static constexpr int Size() { return 4; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (double val) { data = _mm256_set1_pd(val); }
+    SIMD (int val)    { data = _mm256_set1_pd(val); }
+    SIMD (size_t val) { data = _mm256_set1_pd(val); }
+    SIMD (double v0, double v1, double v2, double v3) { data = _mm256_set_pd(v3,v2,v1,v0); }
+    SIMD (SIMD<double,2> v0, SIMD<double,2> v1) : SIMD(v0[0], v0[1], v1[0], v1[1]) { ; }
+    SIMD (double const * p) { data = _mm256_loadu_pd(p); }
+    SIMD (double const * p, SIMD<mask64,4> mask) { data = _mm256_maskload_pd(p, mask.Data()); }
+    SIMD (__m256d _data) { data = _data; }
+    SIMD (std::array<double,4> a)
+      : data{_mm256_set_pd(a[3],a[2],a[1],a[0])}
+    {}
+
+    void Store (double * p) { _mm256_storeu_pd(p, data); }
+    void Store (double * p, SIMD<mask64,4> mask) { _mm256_maskstore_pd(p, mask.Data(), data); }
+
+    template<typename T, typename std::enable_if<std::is_convertible<T, std::function<double(int)>>::value, int>::type = 0>
+    SIMD (const T & func)
+    {
+      data = _mm256_set_pd(func(3), func(2), func(1), func(0));
+    }
+
+    NETGEN_INLINE double operator[] (int i) const { return ((double*)(&data))[i]; }
+    NETGEN_INLINE double & operator[] (int i) { return ((double*)(&data))[i]; }
+    // [[deprecated("don't write to individual elements of SIMD")]]
+    // NETGEN_INLINE double & operator[] (int i) { return ((double*)(&data))[i]; }
+    NETGEN_INLINE __m256d Data() const { return data; }
+    NETGEN_INLINE __m256d & Data() { return data; }
+
+    SIMD<double,2> Lo() const { return _mm256_extractf128_pd(data, 0); }
+    SIMD<double,2> Hi() const { return _mm256_extractf128_pd(data, 1); }
+
+    operator std::tuple<double&,double&,double&,double&> ()
+    { return std::tuple<double&,double&,double&,double&>((*this)[0], (*this)[1], (*this)[2], (*this)[3]); }
+
+    template <int I>
+    double Get() const
+    {
+      static_assert(I>=0 && I<4, "Index out of range");
+      return (*this)[I];
+    }
+  };
+
+  NETGEN_INLINE auto Unpack (SIMD<double,4> a, SIMD<double,4> b)
+  {
+    return std::make_tuple(SIMD<double,4>(_mm256_unpacklo_pd(a.Data(),b.Data())),
+                      SIMD<double,4>(_mm256_unpackhi_pd(a.Data(),b.Data())));
+  }
+
+  NETGEN_INLINE SIMD<double,4> operator- (SIMD<double,4> a) { return _mm256_xor_pd(a.Data(), _mm256_set1_pd(-0.0)); }
+  NETGEN_INLINE SIMD<double,4> operator+ (SIMD<double,4> a, SIMD<double,4> b) { return _mm256_add_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,4> operator- (SIMD<double,4> a, SIMD<double,4> b) { return _mm256_sub_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,4> operator* (SIMD<double,4> a, SIMD<double,4> b) { return _mm256_mul_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,4> operator/ (SIMD<double,4> a, SIMD<double,4> b) { return _mm256_div_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,4> operator* (double a, SIMD<double,4> b) { return _mm256_set1_pd(a)*b.Data(); }
+  NETGEN_INLINE SIMD<double,4> operator* (SIMD<double,4> b, double a) { return _mm256_set1_pd(a)*b.Data(); }
+
+  NETGEN_INLINE SIMD<double,4> sqrt (SIMD<double,4> a) { return _mm256_sqrt_pd(a.Data()); }
+  NETGEN_INLINE SIMD<double,4> floor (SIMD<double,4> a) { return _mm256_floor_pd(a.Data()); }
+  NETGEN_INLINE SIMD<double,4> ceil (SIMD<double,4> a) { return _mm256_ceil_pd(a.Data()); }
+  NETGEN_INLINE SIMD<double,4> fabs (SIMD<double,4> a) { return _mm256_max_pd(a.Data(), (-a).Data()); }
+
+
+#ifdef __FMA__
+  NETGEN_INLINE SIMD<double,4> FMA (SIMD<double,4> a, SIMD<double,4> b, SIMD<double,4> c)
+  {
+    return _mm256_fmadd_pd (a.Data(), b.Data(), c.Data());
+  }
+  NETGEN_INLINE SIMD<double,4> FMA (const double & a, SIMD<double,4> b, SIMD<double,4> c)
+  {
+    return _mm256_fmadd_pd (_mm256_set1_pd(a), b.Data(), c.Data());
+  }
+  NETGEN_INLINE SIMD<double,4> FNMA (SIMD<double,4> a, SIMD<double,4> b, SIMD<double,4> c)
+  {
+    return _mm256_fnmadd_pd (a.Data(), b.Data(), c.Data());
+  }
+  NETGEN_INLINE SIMD<double,4> FNMA (const double & a, SIMD<double,4> b, SIMD<double,4> c)
+  {
+    return _mm256_fnmadd_pd (_mm256_set1_pd(a), b.Data(), c.Data());
+  }
+#endif
+
+#if defined(__FMA__) && !defined(__AVX512F__)
+  // make sure to use the update-version of fma
+  // important in matrix kernels using 12 sum-registers, 3 a-values and updated b-value
+  // avx512 has enough registers, and gcc seems to use only the first 16 z-regs
+  NETGEN_INLINE void FMAasm (SIMD<double,4> a, SIMD<double,4> b, SIMD<double,4> & sum)
+  {
+    asm ("vfmadd231pd %[a], %[b], %[sum]"
+         : [sum] "+x" (sum.Data())
+         : [a] "x" (a.Data()), [b] "x" (b.Data())
+         );
+  }
+
+  NETGEN_INLINE void FNMAasm (SIMD<double,4> a, SIMD<double,4> b, SIMD<double,4> & sum)
+  {
+    asm ("vfnmadd231pd %[a], %[b], %[sum]"
+         : [sum] "+x" (sum.Data())
+         : [a] "x" (a.Data()), [b] "x" (b.Data())
+         );
+  }
+#endif
+
+#if defined(__FMA__) 
+  NETGEN_INLINE SIMD<double,4> FMAddSub (SIMD<double,4> a, SIMD<double,4> b, SIMD<double,4> c)
+  {
+    return _mm256_fmaddsub_pd(a.Data(), b.Data(), c.Data());
+  }
+#endif  
+  
+  NETGEN_INLINE SIMD<double,4> SwapPairs (SIMD<double,4> a)
+  {
+    return _mm256_shuffle_pd (a.Data(), a.Data(), 0b0101);
+  }
+
+  
+  NETGEN_INLINE SIMD<mask64,4> operator<= (SIMD<double,4> a , SIMD<double,4> b)
+  { return _mm256_cmp_pd (a.Data(), b.Data(), _CMP_LE_OQ); }
+  NETGEN_INLINE SIMD<mask64,4> operator< (SIMD<double,4> a , SIMD<double,4> b)
+  { return _mm256_cmp_pd (a.Data(), b.Data(), _CMP_LT_OQ); }
+  NETGEN_INLINE SIMD<mask64,4> operator>= (SIMD<double,4> a , SIMD<double,4> b)
+  { return _mm256_cmp_pd (a.Data(), b.Data(), _CMP_GE_OQ); }
+  NETGEN_INLINE SIMD<mask64,4> operator> (SIMD<double,4> a , SIMD<double,4> b)
+  { return _mm256_cmp_pd (a.Data(), b.Data(), _CMP_GT_OQ); }
+  NETGEN_INLINE SIMD<mask64,4> operator== (SIMD<double,4> a , SIMD<double,4> b)
+  { return _mm256_cmp_pd (a.Data(), b.Data(), _CMP_EQ_OQ); }
+  NETGEN_INLINE SIMD<mask64,4> operator!= (SIMD<double,4> a , SIMD<double,4> b)
+  { return _mm256_cmp_pd (a.Data(), b.Data(), _CMP_NEQ_OQ); }
+
+  NETGEN_INLINE SIMD<mask64,4> operator<= (SIMD<int64_t,4> a , SIMD<int64_t,4> b)
+  { return  _mm256_xor_si256(_mm256_cmpgt_epi64(a.Data(),b.Data()),_mm256_set1_epi32(-1)); }
+  NETGEN_INLINE SIMD<mask64,4> operator< (SIMD<int64_t,4> a , SIMD<int64_t,4> b)
+  { return  my_mm256_cmpgt_epi64(b.Data(),a.Data()); }
+  NETGEN_INLINE SIMD<mask64,4> operator>= (SIMD<int64_t,4> a , SIMD<int64_t,4> b)
+  { return  _mm256_xor_si256(_mm256_cmpgt_epi64(b.Data(),a.Data()),_mm256_set1_epi32(-1)); }
+  NETGEN_INLINE SIMD<mask64,4> operator> (SIMD<int64_t,4> a , SIMD<int64_t,4> b)
+  { return  my_mm256_cmpgt_epi64(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<mask64,4> operator== (SIMD<int64_t,4> a , SIMD<int64_t,4> b)
+  { return  _mm256_cmpeq_epi64(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<mask64,4> operator!= (SIMD<int64_t,4> a , SIMD<int64_t,4> b)
+  { return  _mm256_xor_si256(_mm256_cmpeq_epi64(a.Data(),b.Data()),_mm256_set1_epi32(-1)); }
+
+#ifdef __AVX2__
+  NETGEN_INLINE SIMD<mask64,4> operator&& (SIMD<mask64,4> a, SIMD<mask64,4> b)
+  { return _mm256_and_si256 (a.Data(), b.Data()); }
+  NETGEN_INLINE SIMD<mask64,4> operator|| (SIMD<mask64,4> a, SIMD<mask64,4> b)
+  { return _mm256_or_si256 (a.Data(), b.Data()); }
+  NETGEN_INLINE SIMD<mask64,4> operator! (SIMD<mask64,4> a)
+  { return _mm256_xor_si256 (a.Data(), _mm256_cmpeq_epi64(a.Data(),a.Data())); }
+#else //AVX2 is a superset of AVX. Without it, it is necessary to reinterpret the types
+  NETGEN_INLINE SIMD<mask64,4> operator&& (SIMD<mask64,4> a, SIMD<mask64,4> b)
+  { return _mm256_castpd_si256(_mm256_and_pd (_mm256_castsi256_pd(a.Data()),_mm256_castsi256_pd( b.Data()))); }
+  NETGEN_INLINE SIMD<mask64,4> operator|| (SIMD<mask64,4> a, SIMD<mask64,4> b)
+  { return _mm256_castpd_si256(_mm256_or_pd (_mm256_castsi256_pd(a.Data()), _mm256_castsi256_pd(b.Data()))); }
+  NETGEN_INLINE SIMD<mask64,4> operator! (SIMD<mask64,4> a)
+  { return _mm256_castpd_si256(_mm256_xor_pd (_mm256_castsi256_pd(a.Data()),_mm256_castsi256_pd( _mm256_cmpeq_epi64(a.Data(),a.Data())))); }
+#endif
+  NETGEN_INLINE SIMD<double,4> If (SIMD<mask64,4> a, SIMD<double,4> b, SIMD<double,4> c)
+  { return _mm256_blendv_pd(c.Data(), b.Data(), _mm256_castsi256_pd(a.Data())); }
+
+  NETGEN_INLINE SIMD<double,4> IfPos (SIMD<double,4> a, SIMD<double,4> b, SIMD<double,4> c)
+  {
+    auto cp = _mm256_cmp_pd (a.Data(), _mm256_setzero_pd(), _CMP_GT_OS);
+    return _mm256_blendv_pd(c.Data(), b.Data(), cp);
+  }
+
+  NETGEN_INLINE SIMD<double,4> IfZero (SIMD<double,4> a, SIMD<double,4> b, SIMD<double,4> c)
+  {
+    auto cp = _mm256_cmp_pd (a.Data(), _mm256_setzero_pd(), _CMP_EQ_OS);
+    return _mm256_blendv_pd(c.Data(), b.Data(), cp);
+  }
+
+  NETGEN_INLINE double HSum (SIMD<double,4> sd)
+  {
+    // __m128d hv = _mm_add_pd (_mm256_extractf128_pd(sd.Data(),0), _mm256_extractf128_pd(sd.Data(),1));
+    __m128d hv = (sd.Lo()+sd.Hi()).Data();
+    return _mm_cvtsd_f64 (_mm_hadd_pd (hv, hv));
+  }
+
+  NETGEN_INLINE auto HSum (SIMD<double,4> sd1, SIMD<double,4> sd2)
+  {
+    __m256d hv = _mm256_hadd_pd(sd1.Data(), sd2.Data());
+    __m128d hv2 = _mm_add_pd (_mm256_extractf128_pd(hv,0), _mm256_extractf128_pd(hv,1));
+    return SIMD<double,2>(_mm_cvtsd_f64 (hv2),  _mm_cvtsd_f64(_mm_shuffle_pd (hv2, hv2, 3)));
+  }
+
+  NETGEN_INLINE auto HSum (SIMD<double,4> v1, SIMD<double,4> v2, SIMD<double,4> v3, SIMD<double,4> v4)
+  {
+    __m256d hsum1 = _mm256_hadd_pd (v1.Data(), v2.Data());
+    __m256d hsum2 = _mm256_hadd_pd (v3.Data(), v4.Data());
+    SIMD<double,4> hsum = _mm256_add_pd (_mm256_permute2f128_pd (hsum1, hsum2, 1+2*16),
+                                         _mm256_blend_pd (hsum1, hsum2, 12));
+    return hsum;
+    // return make_tuple(hsum[0], hsum[1], hsum[2], hsum[3]);
+  }
+
+
+  NETGEN_INLINE SIMD<int64_t,4> If (SIMD<mask64,4> a, SIMD<int64_t,4> b, SIMD<int64_t,4> c)
+  { return _mm256_castpd_si256(_mm256_blendv_pd(_mm256_castsi256_pd(c.Data()), _mm256_castsi256_pd(b.Data()),
+                                                _mm256_castsi256_pd(a.Data()))); }
+
+
+
+}
+
+#endif // NETGEN_CORE_SIMD_AVX_HPP
+

libsrc/core/simd_avx512.hpp

@@ -0,0 +1,276 @@
+#ifndef NETGEN_CORE_SIMD_AVX512_HPP
+#define NETGEN_CORE_SIMD_AVX512_HPP
+
+/**************************************************************************/
+/* File:   simd_avx512.hpp                                                */
+/* Author: Joachim Schoeberl, Matthias Hochsteger                         */
+/* Date:   25. Mar. 16                                                    */
+/**************************************************************************/
+
+#include <immintrin.h>
+
+namespace ngcore
+{
+
+  template <>
+  class SIMD<mask64,8>
+  {
+    __mmask8 mask;
+  public:
+    SIMD (size_t i)
+      : mask(_mm512_cmpgt_epi64_mask(_mm512_set1_epi64(i),
+                                     _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0)))
+    { ; }
+    SIMD (int i)
+      : mask(_mm512_cmpgt_epi64_mask(_mm512_set1_epi64(i),
+                                     _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0)))
+    { ; }
+    SIMD (int64_t i)
+      : mask(_mm512_cmpgt_epi64_mask(_mm512_set1_epi64(i),
+                                     _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0)))
+    { ; }
+    SIMD (__mmask8 _mask) : mask(_mask) { ; }
+    __mmask8 Data() const { return mask; }
+    static constexpr int Size() { return 8; }
+    static NETGEN_INLINE SIMD<mask64, 8> GetMaskFromBits (unsigned int i)
+    {
+      return SIMD<mask64, 8>(__mmask8(i));
+    }
+  };
+
+  template<>
+  class alignas(64) SIMD<int64_t,8>
+  {
+    __m512i data;
+
+  public:
+    static constexpr int Size() { return 8; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (int64_t val) { data = _mm512_set1_epi64(val); }
+    SIMD (int64_t v0, int64_t v1, int64_t v2, int64_t v3, int64_t v4, int64_t v5, int64_t v6, int64_t v7) { data = _mm512_set_epi64(v7,v6,v5,v4,v3,v2,v1,v0); }
+    SIMD (__m512i _data) { data = _data; }
+
+    template<typename T, typename std::enable_if<std::is_convertible<T, std::function<int64_t(int)>>::value, int>::type = 0>
+      SIMD (const T & func)
+    {
+      data = _mm512_set_epi64(func(7), func(6), func(5), func(4), func(3), func(2), func(1), func(0));
+    }
+
+
+    NETGEN_INLINE auto operator[] (int i) const { return ((int64_t*)(&data))[i]; }
+    NETGEN_INLINE auto & operator[] (int i) { return ((int64_t*)(&data))[i]; }
+    NETGEN_INLINE __m512i Data() const { return data; }
+    NETGEN_INLINE __m512i & Data() { return data; }
+    static SIMD FirstInt() { return { 0, 1, 2, 3, 4, 5, 6, 7 }; }
+  };
+
+  NETGEN_INLINE SIMD<int64_t,8> operator-(SIMD<int64_t,8> a) { return _mm512_sub_epi64(_mm512_setzero_si512(), a.Data()); }
+
+  NETGEN_INLINE SIMD<int64_t,8> operator+ (SIMD<int64_t,8> a, SIMD<int64_t,8> b) { return _mm512_add_epi64(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<int64_t,8> operator- (SIMD<int64_t,8> a, SIMD<int64_t,8> b) { return _mm512_sub_epi64(a.Data(),b.Data()); }
+
+   NETGEN_INLINE SIMD<int64_t,8> If (SIMD<mask64,8> a, SIMD<int64_t,8> b, SIMD<int64_t,8> c)
+  { return _mm512_mask_blend_epi64(a.Data(), c.Data(), b.Data()); }
+
+
+  template<>
+  class alignas(64) SIMD<double,8>
+  {
+    __m512d data;
+  public:
+    static constexpr int Size() { return 8; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (double val) { data = _mm512_set1_pd(val); }
+    SIMD (int val)    { data = _mm512_set1_pd(val); }
+    SIMD (size_t val) { data = _mm512_set1_pd(val); }
+    SIMD (double const * p) { data = _mm512_loadu_pd(p); }
+    SIMD (double const * p, SIMD<mask64,8> mask)
+      { data = _mm512_mask_loadu_pd(_mm512_setzero_pd(), mask.Data(), p); }
+    SIMD (__m512d _data) { data = _data; }
+    SIMD (SIMD<double,4> v0, SIMD<double,4> v1)
+        : data(_mm512_set_pd(v1[3], v1[2], v1[1], v1[0], v0[3], v0[2], v0[1], v0[0]))
+    {}
+    SIMD (SIMD<double,6> v0, SIMD<double,2> v1)
+        : data(_mm512_set_pd(v1[1], v1[0], v0[5], v0[4], v0[3], v0[2], v0[1], v0[0]))
+    {}
+
+    template<typename T, typename std::enable_if<std::is_convertible<T, std::function<double(int)>>::value, int>::type = 0>
+    SIMD (const T & func)
+    {
+      data = _mm512_set_pd(func(7), func(6), func(5), func(4), func(3), func(2), func(1), func(0));
+    }
+
+    void Store (double * p) { _mm512_storeu_pd(p, data); }
+    void Store (double * p, SIMD<mask64,8> mask) { _mm512_mask_storeu_pd(p, mask.Data(), data); }
+
+    template <typename Function>
+    void SIMD_function (const Function & func, std::true_type)
+    {
+      data = (__m512d){ func(7), func(6), func(5), func(4),
+                       func(3), func(2), func(1), func(0) };
+    }
+
+    // not a function
+    void SIMD_function (double const * p, std::false_type)
+    {
+      data = _mm512_loadu_pd(p);
+    }
+
+    void SIMD_function (double val, std::false_type)
+    {
+      data = _mm512_set1_pd(val);
+    }
+
+    void SIMD_function (__m512d _data, std::false_type)
+    {
+      data = _data;
+    }
+
+    NETGEN_INLINE double operator[] (int i) const { return ((double*)(&data))[i]; }
+    NETGEN_INLINE double & operator[] (int i) { return ((double*)(&data))[i]; }
+    NETGEN_INLINE __m512d Data() const { return data; }
+    NETGEN_INLINE __m512d & Data() { return data; }
+
+    SIMD<double,4> Lo() const { return _mm512_extractf64x4_pd(data, 0); }
+    SIMD<double,4> Hi() const { return _mm512_extractf64x4_pd(data, 1); }
+
+    template <int I>
+    double Get() const
+    {
+      static_assert(I>=0 && I<8, "Index out of range");
+      return (*this)[I];
+    }
+  };
+
+  NETGEN_INLINE SIMD<double,8> operator- (SIMD<double,8> a) { return _mm512_xor_pd(a.Data(), _mm512_set1_pd(-0.0)); } //{ return -a.Data(); }
+  NETGEN_INLINE SIMD<double,8> operator+ (SIMD<double,8> a, SIMD<double,8> b) { return _mm512_add_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,8> operator- (SIMD<double,8> a, SIMD<double,8> b) { return _mm512_sub_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,8> operator* (SIMD<double,8> a, SIMD<double,8> b) { return _mm512_mul_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,8> operator/ (SIMD<double,8> a, SIMD<double,8> b) { return _mm512_div_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,8> operator* (double a, SIMD<double,8> b) { return _mm512_set1_pd(a)*b.Data(); }
+  NETGEN_INLINE SIMD<double,8> operator* (SIMD<double,8> b, double a) { return _mm512_set1_pd(a)*b.Data(); }
+
+  NETGEN_INLINE SIMD<double,8> sqrt (SIMD<double,8> a) { return _mm512_sqrt_pd(a.Data()); }
+  NETGEN_INLINE SIMD<double,8> floor (SIMD<double,8> a) { return _mm512_floor_pd(a.Data()); }
+  NETGEN_INLINE SIMD<double,8> ceil (SIMD<double,8> a) { return _mm512_ceil_pd(a.Data()); }
+  NETGEN_INLINE SIMD<double,8> fabs (SIMD<double,8> a) { return _mm512_max_pd(a.Data(), ( - a).Data()); }
+
+  NETGEN_INLINE SIMD<mask64,8> operator<= (SIMD<double,8> a , SIMD<double,8> b)
+  { return _mm512_cmp_pd_mask (a.Data(), b.Data(), _CMP_LE_OQ); }
+  NETGEN_INLINE SIMD<mask64,8> operator< (SIMD<double,8> a , SIMD<double,8> b)
+  { return _mm512_cmp_pd_mask (a.Data(), b.Data(), _CMP_LT_OQ); }
+  NETGEN_INLINE SIMD<mask64,8> operator>= (SIMD<double,8> a , SIMD<double,8> b)
+  { return _mm512_cmp_pd_mask (a.Data(), b.Data(), _CMP_GE_OQ); }
+  NETGEN_INLINE SIMD<mask64,8> operator> (SIMD<double,8> a , SIMD<double,8> b)
+  { return _mm512_cmp_pd_mask (a.Data(), b.Data(), _CMP_GT_OQ); }
+  NETGEN_INLINE SIMD<mask64,8> operator== (SIMD<double,8> a , SIMD<double,8> b)
+  { return _mm512_cmp_pd_mask (a.Data(), b.Data(), _CMP_EQ_OQ); }
+  NETGEN_INLINE SIMD<mask64,8> operator!= (SIMD<double,8> a , SIMD<double,8> b)
+  { return _mm512_cmp_pd_mask (a.Data(), b.Data(), _CMP_NEQ_OQ); }
+
+  NETGEN_INLINE SIMD<mask64,8> operator<= (SIMD<int64_t,8> a , SIMD<int64_t,8> b)
+  { return _mm512_cmp_epi64_mask (a.Data(), b.Data(), _MM_CMPINT_LE); }
+  NETGEN_INLINE SIMD<mask64,8> operator< (SIMD<int64_t,8> a , SIMD<int64_t,8> b)
+  { return _mm512_cmp_epi64_mask (a.Data(), b.Data(), _MM_CMPINT_LT); }
+  NETGEN_INLINE SIMD<mask64,8> operator>= (SIMD<int64_t,8> a , SIMD<int64_t,8> b)
+  { return _mm512_cmp_epi64_mask (a.Data(), b.Data(),  _MM_CMPINT_NLT); }
+  NETGEN_INLINE SIMD<mask64,8> operator> (SIMD<int64_t,8> a , SIMD<int64_t,8> b)
+  { return _mm512_cmp_epi64_mask (a.Data(), b.Data(), _MM_CMPINT_NLE); }
+  NETGEN_INLINE SIMD<mask64,8> operator== (SIMD<int64_t,8> a , SIMD<int64_t,8> b)
+  { return _mm512_cmp_epi64_mask (a.Data(), b.Data(), _MM_CMPINT_EQ); }
+  NETGEN_INLINE SIMD<mask64,8> operator!= (SIMD<int64_t,8> a , SIMD<int64_t,8> b)
+  { return _mm512_cmp_epi64_mask (a.Data(), b.Data(), _MM_CMPINT_NE); }
+
+  NETGEN_INLINE SIMD<mask64,8> operator&& (SIMD<mask64,8> a, SIMD<mask64,8> b)
+  { return (__mmask8)(a.Data() & b.Data()); }
+  NETGEN_INLINE SIMD<mask64,8> operator|| (SIMD<mask64,8> a, SIMD<mask64,8> b)
+  { return (__mmask8)(a.Data() | b.Data()); }
+  NETGEN_INLINE SIMD<mask64,8> operator! (SIMD<mask64,8> a)
+  { return (__mmask8)(~a.Data()); }
+
+  NETGEN_INLINE SIMD<double,8> If (SIMD<mask64,8> a, SIMD<double,8> b, SIMD<double,8> c)
+  { return _mm512_mask_blend_pd(a.Data(), c.Data(), b.Data()); }
+
+  NETGEN_INLINE SIMD<double,8> IfPos (SIMD<double,8> a, SIMD<double> b, SIMD<double> c)
+  {
+    auto k = _mm512_cmp_pd_mask(a.Data(),_mm512_setzero_pd(), _CMP_GT_OS);
+    return _mm512_mask_blend_pd(k,c.Data(),b.Data());
+  }
+  NETGEN_INLINE SIMD<double,8> IfZero (SIMD<double,8> a, SIMD<double,8> b, SIMD<double,8> c)
+  {
+    auto k = _mm512_cmp_pd_mask(a.Data(),_mm512_setzero_pd(), _CMP_EQ_OS);
+    return _mm512_mask_blend_pd(k,c.Data(),b.Data());
+  }
+
+
+  NETGEN_INLINE auto Unpack (SIMD<double,8> a, SIMD<double,8> b)
+  {
+    return std::make_tuple(SIMD<double,8>(_mm512_unpacklo_pd(a.Data(),b.Data())),
+                      SIMD<double,8>(_mm512_unpackhi_pd(a.Data(),b.Data())));
+  }
+
+
+  NETGEN_INLINE double HSum (SIMD<double,8> sd)
+  {
+    SIMD<double,4> low = _mm512_extractf64x4_pd(sd.Data(),0);
+    SIMD<double,4> high = _mm512_extractf64x4_pd(sd.Data(),1);
+    return HSum(low)+HSum(high);
+  }
+
+  NETGEN_INLINE auto HSum (SIMD<double,8> sd1, SIMD<double,8> sd2)
+  {
+    return SIMD<double,2>(HSum(sd1), HSum(sd2));
+  }
+
+  NETGEN_INLINE SIMD<double,4> HSum (SIMD<double,8> v1, SIMD<double,8> v2, SIMD<double,8> v3, SIMD<double,8> v4)
+  {
+    SIMD<double> lo,hi;
+    std::tie(lo,hi) = Unpack(v1, v2);
+    SIMD<double> sum01 = lo+hi;
+    std::tie(lo,hi) = Unpack(v3, v4);
+    SIMD<double> sum23 = lo+hi;
+    // sum01  b a b a b a b a
+    // sum23  d c d c d c d c
+    // __m512 perm = _mm512_permutex2var_pd (sum01.Data(), _mm512_set_epi64(1,2,3,4,5,6,7,8), sum23.Data());
+    SIMD<double,4> ab =  _mm512_extractf64x4_pd(sum01.Data(),0) + _mm512_extractf64x4_pd(sum01.Data(),1);
+    SIMD<double,4> cd =  _mm512_extractf64x4_pd(sum23.Data(),0) + _mm512_extractf64x4_pd(sum23.Data(),1);
+    return _mm256_add_pd (_mm256_permute2f128_pd (ab.Data(), cd.Data(), 1 + 2 * 16), _mm256_blend_pd(ab.Data(), cd.Data(), 12));
+  }
+
+  NETGEN_INLINE SIMD<double,8> FMA (SIMD<double,8> a, SIMD<double,8> b, SIMD<double,8> c)
+  {
+    return _mm512_fmadd_pd (a.Data(), b.Data(), c.Data());
+  }
+  NETGEN_INLINE SIMD<double,8> FMA (const double & a, SIMD<double,8> b, SIMD<double,8> c)
+  {
+    return _mm512_fmadd_pd (_mm512_set1_pd(a), b.Data(), c.Data());
+  }
+
+  NETGEN_INLINE SIMD<double,8> FNMA (SIMD<double,8> a, SIMD<double,8> b, SIMD<double,8> c)
+  {
+    return _mm512_fnmadd_pd (a.Data(), b.Data(), c.Data());
+  }
+  NETGEN_INLINE SIMD<double,8> FNMA (const double & a, SIMD<double,8> b, SIMD<double,8> c)
+  {
+    return _mm512_fnmadd_pd (_mm512_set1_pd(a), b.Data(), c.Data());
+  }
+
+  NETGEN_INLINE SIMD<double,8> FMAddSub (SIMD<double,8> a, SIMD<double,8> b, SIMD<double,8> c)
+  {
+    return _mm512_fmaddsub_pd(a.Data(), b.Data(), c.Data());
+  }
+
+  NETGEN_INLINE SIMD<double,8> SwapPairs (SIMD<double,8> a)
+  {
+    return _mm512_shuffle_pd (a.Data(), a.Data(), 0b01010101);
+  }
+  
+}
+
+#endif // NETGEN_CORE_SIMD_AVX512_HPP

libsrc/core/simd_generic.hpp

@@ -0,0 +1,791 @@
+#ifndef NETGEN_CORE_SIMD_GENERIC_HPP
+#define NETGEN_CORE_SIMD_GENERIC_HPP
+
+/**************************************************************************/
+/* File:   simd_base.hpp                                                  */
+/* Author: Joachim Schoeberl, Matthias Hochsteger                         */
+/* Date:   25. Mar. 16                                                    */
+/**************************************************************************/
+
+#include <type_traits>
+#include <functional>
+#include <tuple>
+#include <cmath>
+
+#include "array.hpp"
+
+namespace ngcore
+{
+#if defined __AVX512F__
+    #define NETGEN_DEFAULT_SIMD_SIZE 8
+#elif defined __AVX__
+    #define NETGEN_DEFAULT_SIMD_SIZE 4
+#else
+    #define NETGEN_DEFAULT_SIMD_SIZE 2
+#endif
+
+  constexpr int GetDefaultSIMDSize() {
+      return NETGEN_DEFAULT_SIMD_SIZE;
+  }
+
+  constexpr bool IsNativeSIMDSize(int n) {
+    if(n==1) return true;
+    if(n==2) return true;
+#if defined __AVX__
+    if(n==4) return true;
+#endif
+#if defined __AVX512F__
+    if(n==8) return true;
+#endif
+    return false;
+  }
+
+  // split n = k+l such that k is the largest natively supported simd size < n
+  constexpr int GetLargestNativeSIMDPart(int n) {
+      int k = n-1;
+      while(!IsNativeSIMDSize(k))
+          k--;
+      return k;
+  }
+
+
+  template <typename T, int N=GetDefaultSIMDSize()> class SIMD;
+
+  class mask64;
+
+  ////////////////////////////////////////////////////////////////////////////
+  namespace detail {
+    template <typename T, size_t N, size_t... I>
+    auto array_range_impl(std::array<T, N> const& arr,
+                   size_t first,
+                   std::index_sequence<I...>)
+    -> std::array<T, sizeof...(I)> {
+        return {arr[first + I]...};
+    }
+
+    template <size_t S, typename T, size_t N>
+    auto array_range(std::array<T, N> const& arr, size_t first) {
+      return array_range_impl(arr, first, std::make_index_sequence<S>{});
+    }
+  
+  } // namespace detail
+
+  ////////////////////////////////////////////////////////////////////////////
+  // mask
+
+  template <>
+  class SIMD<mask64,1>
+  {
+    int64_t mask;
+  public:
+    SIMD (int64_t i)
+      : mask(i > 0 ? -1 : 0) { ; }
+    bool Data() const { return mask; }
+    static constexpr int Size() { return 1; }
+    auto operator[] (int /* i */) const { return mask; }
+  };
+
+
+  template <int N>
+  class alignas(GetLargestNativeSIMDPart(N)*sizeof(int64_t)) SIMD<mask64,N>
+  {
+    static constexpr int N1 = GetLargestNativeSIMDPart(N);
+    static constexpr int N2 = N-N1;
+
+    SIMD<mask64,N1> lo;
+    SIMD<mask64,N2> hi;
+  public:
+
+    SIMD (int64_t i) : lo(i), hi(i-N1 ) { ; }
+    SIMD (SIMD<mask64,N1> lo_, SIMD<mask64,N2> hi_) : lo(lo_), hi(hi_) { ; }
+    SIMD<mask64,N1> Lo() const { return lo; }
+    SIMD<mask64,N2> Hi() const { return hi; }
+    static constexpr int Size() { return N; }
+  };
+
+  template<int N>
+  NETGEN_INLINE SIMD<mask64,N> operator&& (SIMD<mask64,N> a, SIMD<mask64,N> b)
+    {
+      if constexpr(N==1) return a.Data() && b.Data();
+      else               return { a.Lo() && b.Lo(), a.Hi() && b.Hi() };
+    }
+
+
+  ////////////////////////////////////////////////////////////////////////////
+  // int64
+
+  template<>
+  class SIMD<int64_t,1>
+  {
+    int64_t data;
+
+  public:
+    static constexpr int Size() { return 1; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD & operator= (const SIMD &) = default;
+    SIMD (int val) : data{val} {}
+    SIMD (int64_t val) : data{val} {}
+    SIMD (size_t val) : data(val) {}
+    explicit SIMD (std::array<int64_t, 1> arr)
+        : data{arr[0]}
+    {}
+
+    int64_t operator[] (int i) const { return ((int64_t*)(&data))[i]; }
+    auto Data() const { return data; }
+    static SIMD FirstInt(int64_t n0=0) { return {n0}; }
+    template <int I>
+    int64_t Get()
+    {
+      static_assert(I==0);
+      return data;
+    }
+  };
+
+  template<int N>
+  class alignas(GetLargestNativeSIMDPart(N)*sizeof(int64_t)) SIMD<int64_t,N>
+  {
+    static constexpr int N1 = GetLargestNativeSIMDPart(N);
+    static constexpr int N2 = N-N1;
+
+    SIMD<int64_t,N1> lo;
+    SIMD<int64_t,N2> high;
+
+  public:
+    static constexpr int Size() { return N; }
+
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (int val) : lo{val}, high{val} { ; }
+    SIMD (int64_t val) : lo{val}, high{val} { ; }
+    SIMD (size_t val) : lo{val}, high{val} { ; }
+    SIMD (SIMD<int64_t,N1> lo_, SIMD<int64_t,N2> high_) : lo(lo_), high(high_) { ; }
+
+    explicit SIMD( std::array<int64_t, N> arr )
+        : lo(detail::array_range<N1>(arr, 0)),
+          high(detail::array_range<N2>(arr, N1))
+      {}
+
+    template<typename ...T>
+    explicit SIMD(const T... vals)
+    : lo(detail::array_range<N1>(std::array<int64_t, N>{vals...}, 0)),
+      high(detail::array_range<N2>(std::array<int64_t, N>{vals...}, N1))
+      {
+        static_assert(sizeof...(vals)==N, "wrong number of arguments");
+      }
+
+
+    template<typename T, typename std::enable_if<std::is_convertible<T, std::function<int64_t(int)>>::value, int>::type = 0>
+      SIMD (const T & func)
+    {
+      for(auto i : IntRange(N1))
+          lo[i] = func(i);
+      for(auto i : IntRange(N2))
+          high[i] = func(N1+i);
+    }
+
+    auto Lo() const { return lo; }
+    auto Hi() const { return high; }
+
+    int64_t operator[] (int i) const { return ((int64_t*)(&lo))[i]; }
+
+    /*
+    operator tuple<int64_t&,int64_t&,int64_t&,int64_t&> ()
+    { return tuple<int64_t&,int64_t&,int64_t&,int64_t&>((*this)[0], (*this)[1], (*this)[2], (*this)[3]); }
+    */
+
+    /*
+    static SIMD FirstInt() { return { 0, 1, 2, 3 }; }
+    */
+    static SIMD FirstInt(int64_t n0=0) { return {SIMD<int64_t,N1>::FirstInt(n0), SIMD<int64_t,N2>::FirstInt(n0+N1)}; }
+    template <int I>
+    int64_t Get()
+    {
+      static_assert(I>=0 && I<N, "Index out of range");
+      if constexpr(I<N1) return lo.template Get<I>();
+      else               return high.template Get<I-N1>();
+    }
+  };
+
+
+  ////////////////////////////////////////////////////////////////////////////
+  // double
+
+  template<>
+  class SIMD<double,1>
+  {
+    double data;
+
+  public:
+    static constexpr int Size() { return 1; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD & operator= (const SIMD &) = default;
+    SIMD (double val) { data = val; }
+    SIMD (int val)    { data = val; }
+    SIMD (size_t val) { data = val; }
+    SIMD (double const * p) { data = *p; }
+    SIMD (double const * p, SIMD<mask64,1> mask) { data = mask.Data() ? *p : 0.0; }
+    explicit SIMD (std::array<double, 1> arr)
+        : data{arr[0]}
+    {}
+
+    template <typename T, typename std::enable_if<std::is_convertible<T,std::function<double(int)>>::value,int>::type = 0>
+    SIMD (const T & func)
+    {
+      data = func(0);
+    }
+
+    template <typename T, typename std::enable_if<std::is_convertible<T,std::function<double(int)>>::value,int>::type = 0>
+    SIMD & operator= (const T & func)
+    {
+      data = func(0);
+      return *this;
+    }
+
+    void Store (double * p) { *p = data; }
+    void Store (double * p, SIMD<mask64,1> mask) { if (mask.Data()) *p = data; }
+
+    double operator[] (int i) const { return ((double*)(&data))[i]; }
+    double Data() const { return data; }
+    template <int I>
+    double Get()
+    {
+      static_assert(I==0);
+      return data;
+    }
+  };
+
+
+  template<int N>
+  class alignas(GetLargestNativeSIMDPart(N)*sizeof(double)) SIMD<double, N>
+  {
+    static constexpr int N1 = GetLargestNativeSIMDPart(N);
+    static constexpr int N2 = N-N1;
+
+    SIMD<double, N1> lo;
+    SIMD<double, N2> high;
+
+  public:
+    static constexpr int Size() { return N; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD (SIMD<double,N1> lo_, SIMD<double,N2> hi_) : lo(lo_), high(hi_) { ; }
+
+    template <typename T, typename std::enable_if<std::is_convertible<T,std::function<double(int)>>::value,int>::type = 0>
+    SIMD (const T & func)
+    {
+      double  *p = (double*)this;
+      for(auto i : IntRange(N))
+          p[i] = func(i);
+    }
+
+    template <typename T, typename std::enable_if<std::is_convertible<T,std::function<double(int)>>::value,int>::type = 0>
+    SIMD & operator= (const T & func)
+    {
+      double  *p = (double*)this;
+      for(auto i : IntRange(N))
+          p[i] = func(i);
+      return *this;
+    }
+
+
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (double val) : lo{val}, high{val} { ; }
+    SIMD (int val)    : lo{val}, high{val} { ; }
+    SIMD (size_t val) : lo{val}, high{val} { ; }
+
+    SIMD (double const * p) : lo{p}, high{p+N1} { ; }
+    SIMD (double const * p, SIMD<mask64,N> mask)
+        : lo{p, mask.Lo()}, high{p+N1, mask.Hi()}
+      { }
+    SIMD (double * p) : lo{p}, high{p+N1} { ; }
+    SIMD (double * p, SIMD<mask64,N> mask)
+        : lo{p, mask.Lo()}, high{p+N1, mask.Hi()}
+      { }
+
+    explicit SIMD( std::array<double, N> arr )
+        : lo(detail::array_range<N1>(arr, 0)),
+          high(detail::array_range<N2>(arr, N1))
+      {}
+
+    template<typename ...T>
+    explicit SIMD(const T... vals)
+    : lo(detail::array_range<N1>(std::array<double, N>{vals...}, 0)),
+      high(detail::array_range<N2>(std::array<double, N>{vals...}, N1))
+      {
+        static_assert(sizeof...(vals)==N, "wrong number of arguments");
+      }
+
+    void Store (double * p) { lo.Store(p); high.Store(p+N1); }
+    void Store (double * p, SIMD<mask64,N> mask)
+    {
+      lo.Store(p, mask.Lo());
+      high.Store(p+N1, mask.Hi());
+    }
+
+    auto Lo() const { return lo; }
+    auto Hi() const { return high; }
+
+    double operator[] (int i) const { return ((double*)(&lo))[i]; }
+
+    template<typename=std::enable_if<N==2>>
+    operator std::tuple<double&,double&> ()
+    { 
+	double *p = (double*)this;
+	return std::tuple<double&,double&>(p[0], p[1]); 
+    }
+
+    template<typename=std::enable_if<N==4>>
+    operator std::tuple<double&,double&,double&,double&> ()
+    { return std::tuple<double&,double&,double&,double&>((*this)[0], (*this)[1], (*this)[2], (*this)[3]); }
+
+    template <int I>
+    double Get()
+    {
+      static_assert(I>=0 && I<N, "Index out of range");
+      if constexpr(I<N1) return lo.template Get<I>();
+      else               return high.template Get<I-N1>();
+    }
+    auto Data() const { return *this; }
+  };
+
+
+  // Generic operators for any arithmetic type/simd width
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<T,N> operator+ (SIMD<T,N> a, SIMD<T,N> b) {
+      if constexpr(N==1) return a.Data()+b.Data();
+      else               return { a.Lo()+b.Lo(), a.Hi()+b.Hi() };
+  }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<T,N> operator- (SIMD<T,N> a, SIMD<T,N> b) {
+      if constexpr(N==1) return a.Data()-b.Data();
+      else               return { a.Lo()-b.Lo(), a.Hi()-b.Hi() };
+  }
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<T,N> operator- (SIMD<T,N> a) {
+      if constexpr(N==1) return -a.Data();
+      else               return { -a.Lo(), -a.Hi() };
+  }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<T,N> operator* (SIMD<T,N> a, SIMD<T,N> b) {
+      if constexpr(N==1) return a.Data()*b.Data();
+      else               return { a.Lo()*b.Lo(), a.Hi()*b.Hi() };
+  }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<T,N> operator/ (SIMD<T,N> a, SIMD<T,N> b) {
+      if constexpr(N==1) return a.Data()/b.Data();
+      else               return { a.Lo()/b.Lo(), a.Hi()/b.Hi() };
+  }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<mask64,N> operator< (SIMD<T,N> a, SIMD<T,N> b)
+    {
+      if constexpr(N==1) return a.Data() < b.Data();
+      else               return { a.Lo()<b.Lo(), a.Hi()<b.Hi() };
+    }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<mask64,N> operator<= (SIMD<T,N> a, SIMD<T,N> b)
+    {
+      if constexpr(N==1) return a.Data() <= b.Data();
+      else               return { a.Lo()<=b.Lo(), a.Hi()<=b.Hi() };
+    }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<mask64,N> operator> (SIMD<T,N> a, SIMD<T,N> b)
+    {
+      if constexpr(N==1) return a.Data() > b.Data();
+      else               return { a.Lo()>b.Lo(), a.Hi()>b.Hi() };
+    }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<mask64,N> operator>= (SIMD<T,N> a, SIMD<T,N> b)
+    {
+      if constexpr(N==1) return a.Data() >= b.Data();
+      else               return { a.Lo()>=b.Lo(), a.Hi()>=b.Hi() };
+    }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<mask64,N> operator== (SIMD<T,N> a, SIMD<T,N> b)
+    {
+      if constexpr(N==1) return a.Data() == b.Data();
+      else               return { a.Lo()==b.Lo(), a.Hi()==b.Hi() };
+    }
+
+  template <typename T, int N>
+  NETGEN_INLINE SIMD<mask64,N> operator!= (SIMD<T,N> a, SIMD<T,N> b)
+    {
+      if constexpr(N==1) return a.Data() != b.Data();
+      else               return { a.Lo()!=b.Lo(), a.Hi()!=b.Hi() };
+    }
+
+  // int64_t operators with scalar operand (implement overloads to allow implicit casts for second operand)
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator+ (SIMD<int64_t,N> a, int64_t b) { return a+SIMD<int64_t,N>(b); }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator+ (int64_t a, SIMD<int64_t,N> b) { return SIMD<int64_t,N>(a)+b; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator- (int64_t a, SIMD<int64_t,N> b) { return SIMD<int64_t,N>(a)-b; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator- (SIMD<int64_t,N> a, int64_t b) { return a-SIMD<int64_t,N>(b); }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator* (int64_t a, SIMD<int64_t,N> b) { return SIMD<int64_t,N>(a)*b; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator* (SIMD<int64_t,N> b, int64_t a) { return SIMD<int64_t,N>(a)*b; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator/ (SIMD<int64_t,N> a, int64_t b) { return a/SIMD<int64_t,N>(b); }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> operator/ (int64_t a, SIMD<int64_t,N> b) { return SIMD<int64_t,N>(a)/b; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> & operator+= (SIMD<int64_t,N> & a, SIMD<int64_t,N> b) { a=a+b; return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> & operator+= (SIMD<int64_t,N> & a, int64_t b) { a+=SIMD<int64_t,N>(b); return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> & operator-= (SIMD<int64_t,N> & a, SIMD<int64_t,N> b) { a = a-b; return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> & operator-= (SIMD<int64_t,N> & a, int64_t b) { a-=SIMD<int64_t,N>(b); return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> & operator*= (SIMD<int64_t,N> & a, SIMD<int64_t,N> b) { a=a*b; return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> & operator*= (SIMD<int64_t,N> & a, int64_t b) { a*=SIMD<int64_t,N>(b); return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<int64_t,N> & operator/= (SIMD<int64_t,N> & a, SIMD<int64_t,N> b) { a = a/b; return a; }
+
+  // double operators with scalar operand (implement overloads to allow implicit casts for second operand)
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator+ (SIMD<double,N> a, double b) { return a+SIMD<double,N>(b); }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator+ (double a, SIMD<double,N> b) { return SIMD<double,N>(a)+b; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator- (double a, SIMD<double,N> b) { return SIMD<double,N>(a)-b; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator- (SIMD<double,N> a, double b) { return a-SIMD<double,N>(b); }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator* (double a, SIMD<double,N> b) { return SIMD<double,N>(a)*b; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator* (SIMD<double,N> b, double a) { return SIMD<double,N>(a)*b; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator/ (SIMD<double,N> a, double b) { return a/SIMD<double,N>(b); }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> operator/ (double a, SIMD<double,N> b) { return SIMD<double,N>(a)/b; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> & operator+= (SIMD<double,N> & a, SIMD<double,N> b) { a=a+b; return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> & operator+= (SIMD<double,N> & a, double b) { a+=SIMD<double,N>(b); return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> & operator-= (SIMD<double,N> & a, SIMD<double,N> b) { a = a-b; return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> & operator-= (SIMD<double,N> & a, double b) { a-=SIMD<double,N>(b); return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> & operator*= (SIMD<double,N> & a, SIMD<double,N> b) { a=a*b; return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> & operator*= (SIMD<double,N> & a, double b) { a*=SIMD<double,N>(b); return a; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> & operator/= (SIMD<double,N> & a, SIMD<double,N> b) { a = a/b; return a; }
+
+  // double functions
+
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> L2Norm2 (SIMD<double,N> a) { return a*a; }
+  template <int N>
+  NETGEN_INLINE SIMD<double,N> Trans (SIMD<double,N> a) { return a; }
+
+  template <int N>
+  NETGEN_INLINE double HSum (SIMD<double,N> a)
+    {
+      if constexpr(N==1)
+          return a.Data();
+      else
+          return HSum(a.Lo()) + HSum(a.Hi());
+    }
+
+
+  template<typename T, int N>
+  NETGEN_INLINE SIMD<T,N> IfPos (SIMD<T,N> a, SIMD<T,N> b, SIMD<T,N> c)
+    {
+      if constexpr(N==1) return a.Data()>0.0 ? b : c;
+      else               return { IfPos(a.Lo(), b.Lo(), c.Lo()), IfPos(a.Hi(), b.Hi(), c.Hi())};
+
+    }
+
+  template<typename T, int N>
+  NETGEN_INLINE SIMD<T,N> IfZero (SIMD<T,N> a, SIMD<T,N> b, SIMD<T,N> c)
+    {
+      if constexpr(N==1) return a.Data()==0.0 ? b : c;
+      else               return { IfZero(a.Lo(), b.Lo(), c.Lo()), IfZero(a.Hi(), b.Hi(), c.Hi())};
+
+    }
+
+  template<typename T, int N>
+  NETGEN_INLINE SIMD<T,N> If (SIMD<mask64,N> a, SIMD<T,N> b, SIMD<T,N> c)
+    {
+      if constexpr(N==1) return a.Data() ? b : c;
+      else               return { If(a.Lo(), b.Lo(), c.Lo()), If(a.Hi(), b.Hi(), c.Hi())};
+
+    }
+
+  // a*b+c
+  template <typename T1, typename T2, typename T3>
+  NETGEN_INLINE auto FMA(T1 a, T2 b, T3 c)
+  {
+    return c+a*b;
+  }
+
+  template <typename T1, typename T2, typename T3>
+  NETGEN_INLINE auto FNMA(T1 a, T2 b, T3 c)
+  {
+    return c-a*b;
+  }
+
+  // update form of fma
+  template <int N>
+  void FMAasm (SIMD<double,N> a, SIMD<double,N> b, SIMD<double,N> & sum)
+  {
+    sum = FMA(a,b,sum);
+  }
+
+  // update form of fms
+  template <int N>
+  void FNMAasm (SIMD<double,N> a, SIMD<double,N> b, SIMD<double,N> & sum)
+  {
+    // sum -= a*b;
+    sum = FNMA(a,b,sum);
+  }
+
+  // c += a*b    (a0re, a0im, a1re, a1im, ...), 
+  template <int N>
+  void FMAComplex (SIMD<double,N> a, SIMD<double,N> b, SIMD<double,N> & c)
+  {
+    auto [are, aim] = Unpack(a, a);
+    SIMD<double,N> bswap = SwapPairs(b);
+    SIMD<double,N> aim_bswap = aim*bswap;
+    c += FMAddSub (are, b, aim_bswap);
+  }
+  
+  template <int i, typename T, int N>
+  T get(SIMD<T,N> a) { return a.template Get<i>(); }
+
+  template <int NUM, typename FUNC>
+  NETGEN_INLINE void Iterate2 (FUNC f)
+  {
+    if constexpr (NUM > 1) Iterate2<NUM-1> (f);
+    if constexpr (NUM >= 1) f(std::integral_constant<int,NUM-1>());
+  }
+
+
+  template <typename T, int N>
+  ostream & operator<< (ostream & ost, SIMD<T,N> simd)
+  {
+    /*
+    ost << simd[0];
+    for (int i = 1; i < simd.Size(); i++)
+      ost << " " << simd[i];
+    */
+    Iterate2<simd.Size()> ([&] (auto I) {
+        if (I.value != 0) ost << " ";
+        ost << get<I.value>(simd);
+      });
+    return ost;
+  }
+
+  using std::sqrt;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> sqrt (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return sqrt(a[i]); } );
+  }
+
+  using std::fabs;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> fabs (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return fabs(a[i]); } );
+  }
+
+  using std::floor;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> floor (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return floor(a[i]); } );
+  }
+
+  using std::ceil;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> ceil (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return ceil(a[i]); } );
+  }
+
+  using std::exp;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> exp (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return exp(a[i]); } );
+  }
+
+  using std::log;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> log (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return log(a[i]); } );
+  }
+
+  using std::erf;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> erf (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return erf(a[i]); } );
+  }
+
+  using std::pow;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> pow (ngcore::SIMD<double,N> a, double x) {
+    return ngcore::SIMD<double,N>([a,x](int i)->double { return pow(a[i],x); } );
+  }
+
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> pow (ngcore::SIMD<double,N> a, ngcore::SIMD<double,N> b) {
+    return ngcore::SIMD<double,N>([a,b](int i)->double { return pow(a[i],b[i]); } );
+  }
+
+  using std::sin;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> sin (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return sin(a[i]); } );
+  }
+
+  using std::cos;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> cos (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return cos(a[i]); } );
+  }
+
+  using std::tan;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> tan (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return tan(a[i]); } );
+  }
+
+  using std::atan;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> atan (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return atan(a[i]); } );
+  }
+
+  using std::atan2;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> atan2 (ngcore::SIMD<double,N> y, ngcore::SIMD<double,N> x) {
+    return ngcore::SIMD<double,N>([y,x](int i)->double { return atan2(y[i], x[i]); } );
+  }
+
+  using std::acos;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> acos (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return acos(a[i]); } );
+  }
+
+  using std::asin;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> asin (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return asin(a[i]); } );
+  }
+
+  using std::sinh;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> sinh (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return sinh(a[i]); } );
+  }
+
+  using std::cosh;
+  template <int N>
+  NETGEN_INLINE ngcore::SIMD<double,N> cosh (ngcore::SIMD<double,N> a) {
+    return ngcore::SIMD<double,N>([a](int i)->double { return cosh(a[i]); } );
+  }
+
+  template<int N, typename T>
+  using MultiSIMD = SIMD<T, N*GetDefaultSIMDSize()>;
+
+  template<int N>
+  NETGEN_INLINE auto Unpack (SIMD<double,N> a, SIMD<double,N> b)
+  {
+    if constexpr(N==1)
+      {
+        return std::make_tuple(SIMD<double,N>{a.Data()}, SIMD<double,N>{b.Data()} );
+      }
+    else if constexpr(N==2)
+      {
+        return std::make_tuple(SIMD<double,N>{ a.Lo(), b.Lo() },
+            SIMD<double,N>{ a.Hi(), b.Hi() });
+      }
+    else
+      {
+        auto [a1,b1] = Unpack(a.Lo(), b.Lo());
+        auto [a2,b2] = Unpack(a.Hi(), b.Hi());
+        return std::make_tuple(SIMD<double,N>{ a1, a2 },
+            SIMD<double,N>{ b1, b2 });
+      }
+  }
+
+  // TODO: specialize for AVX, ... 
+  template<int N>
+  NETGEN_INLINE auto SwapPairs (SIMD<double,N> a)
+  {
+    if constexpr(N==1) {
+        // static_assert(false);
+        return a;
+      }
+    else if constexpr(N==2) {
+        return SIMD<double,N> (a.Hi(), a.Lo());
+      }
+    else {
+      return SIMD<double,N> (SwapPairs(a.Lo()), SwapPairs(a.Hi()));
+    }
+  }
+
+
+  template<int N>
+  NETGEN_INLINE auto HSum128 (SIMD<double,N> a)
+  {
+    if constexpr(N==1) {
+        // static_assert(false);
+        return a;
+      }
+    else if constexpr(N==2) {
+        return a;
+      }
+    else {
+      return HSum128(a.Lo()) + HSum128(a.Hi());
+    }
+  }
+
+  
+  // TODO: specialize for AVX, ... 
+  // a*b+-c   (even: -, odd: +)
+  template<int N>
+  NETGEN_INLINE auto FMAddSub (SIMD<double,N> a, SIMD<double,N> b, SIMD<double,N> c)
+  {
+    if constexpr(N==1) {
+        // static_assert(false);
+        return a*b-c;
+      }
+    else if constexpr(N==2) {
+        return SIMD<double,N> (a.Lo()*b.Lo()-c.Lo(),
+                               a.Hi()*b.Hi()+c.Hi());
+      }
+    else {
+      return SIMD<double,N> (FMAddSub(a.Lo(), b.Lo(), c.Lo()),
+                             FMAddSub(a.Hi(), b.Hi(), c.Hi()));
+    }
+  }
+}
+
+
+namespace std
+{
+  // structured binding support
+  template <typename T, int N >
+  struct tuple_size<ngcore::SIMD<T,N>> : std::integral_constant<std::size_t, N> {};
+  template<size_t N, typename T, int M> struct tuple_element<N,ngcore::SIMD<T,M>> { using type = T; };
+}
+
+#endif // NETGEN_CORE_SIMD_GENERIC_HPP

libsrc/core/simd_sse.hpp

@@ -0,0 +1,279 @@
+#ifndef NETGEN_CORE_SIMD_SSE_HPP
+#define NETGEN_CORE_SIMD_SSE_HPP
+
+/**************************************************************************/
+/* File:   simd_sse.hpp                                                   */
+/* Author: Joachim Schoeberl, Matthias Hochsteger                         */
+/* Date:   25. Mar. 16                                                    */
+/**************************************************************************/
+
+#include <immintrin.h>
+
+namespace ngcore
+{
+
+  template <>
+  class SIMD<mask64,2>
+  {
+    __m128i mask;
+  public:
+    SIMD (int i)
+      : mask(_mm_cmpgt_epi32(_mm_set1_epi32(i),
+                             _mm_set_epi32(1, 1, 0, 0)))
+    { ; }
+    SIMD (__m128i _mask) : mask(_mask) { ; }
+    __m128i Data() const { return mask; }
+    static constexpr int Size() { return 2; }
+    static NETGEN_INLINE SIMD<mask64, 2> GetMaskFromBits (unsigned int i);
+  };
+
+  static SIMD<mask64, 2> masks_from_2bits[4] = {
+    _mm_set_epi32 (0,0,0,0), _mm_set_epi32 (0,0,-1,0),
+    _mm_set_epi32 (-1,0,0,0), _mm_set_epi32 (-1,0,-1,0),
+  };
+
+  NETGEN_INLINE SIMD<mask64, 2> SIMD<mask64, 2> :: GetMaskFromBits (unsigned int i)
+  {
+    return masks_from_2bits[i & 3];
+  }
+
+
+  template<>
+  class alignas(16) SIMD<int64_t,2>
+  {
+    __m128i data;
+
+  public:
+    static constexpr int Size() { return 2; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD (int64_t v0, int64_t v1) { data = _mm_set_epi64x(v1,v0); }
+    SIMD (std::array<int64_t, 2> arr)
+        : data{_mm_set_epi64x(arr[1],arr[0])}
+    {}
+
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (int64_t val) { data = _mm_set1_epi64x(val); }
+    SIMD (__m128i _data) { data = _data; }
+
+    template<typename T, typename std::enable_if<std::is_convertible<T, std::function<int64_t(int)>>::value, int>::type = 0>
+    SIMD (const T & func)
+    {
+      data = _mm_set_epi64(func(1), func(0));
+    }
+
+    NETGEN_INLINE auto operator[] (int i) const { return ((int64_t*)(&data))[i]; }
+    NETGEN_INLINE __m128i Data() const { return data; }
+    NETGEN_INLINE __m128i & Data() { return data; }
+    static SIMD FirstInt(int n0=0) { return { n0, n0+1 }; }
+  };
+
+
+
+NETGEN_INLINE SIMD<int64_t,2> operator-(SIMD<int64_t,2> a) { return _mm_sub_epi64(_mm_setzero_si128(), a.Data()); }
+NETGEN_INLINE SIMD<int64_t,2> operator+ (SIMD<int64_t,2> a, SIMD<int64_t,2> b) { return _mm_add_epi64(a.Data(),b.Data()); }
+NETGEN_INLINE SIMD<int64_t,2> operator- (SIMD<int64_t,2> a, SIMD<int64_t,2> b) { return _mm_sub_epi64(a.Data(),b.Data()); }
+
+
+  template<>
+  class alignas(16) SIMD<double,2>
+  {
+    __m128d data;
+
+  public:
+    static constexpr int Size() { return 2; }
+    SIMD () {}
+    SIMD (const SIMD &) = default;
+    SIMD (double v0, double v1) { data = _mm_set_pd(v1,v0); }
+    SIMD (SIMD<double,1> v0, SIMD<double,1> v1)
+        : data{_mm_set_pd(v0.Data(), v1.Data())}
+    { }
+    SIMD (std::array<double, 2> arr)
+        : data{_mm_set_pd(arr[1], arr[0])}
+    {}
+
+    SIMD & operator= (const SIMD &) = default;
+
+    SIMD (double val) { data = _mm_set1_pd(val); }
+    SIMD (int val)    { data = _mm_set1_pd(val); }
+    SIMD (size_t val) { data = _mm_set1_pd(val); }
+
+    SIMD (double const * p) { data = _mm_loadu_pd(p); }
+    SIMD (double const * p, SIMD<mask64,2> mask)
+      {
+#ifdef __AVX__
+        data = _mm_maskload_pd(p, mask.Data());
+#else
+        // this versions segfaults if p points to the last allowed element
+        // happened on Mac with the new SparseCholesky-factorization
+        // data = _mm_and_pd(_mm_castsi128_pd(mask.Data()), _mm_loadu_pd(p));
+        auto pmask = (int64_t*)&mask;
+        data = _mm_set_pd (pmask[1] ? p[1] : 0.0, pmask[0] ? p[0] : 0.0);
+#endif
+      }
+    SIMD (__m128d _data) { data = _data; }
+
+    void Store (double * p) { _mm_storeu_pd(p, data); }
+    void Store (double * p, SIMD<mask64,2> mask)
+    {
+#ifdef __AVX__
+      _mm_maskstore_pd(p, mask.Data(), data);
+#else
+      /*
+      _mm_storeu_pd (p, _mm_or_pd (_mm_and_pd(_mm_castsi128_pd(mask.Data()), data),
+                                   _mm_andnot_pd(_mm_castsi128_pd(mask.Data()), _mm_loadu_pd(p))));
+      */
+      auto pmask = (int64_t*)&mask;
+      if (pmask[0]) p[0] = (*this)[0];
+      if (pmask[1]) p[1] = (*this)[1];
+#endif
+    }
+
+    template<typename T, typename std::enable_if<std::is_convertible<T, std::function<double(int)>>::value, int>::type = 0>
+    SIMD (const T & func)
+    {
+      data = _mm_set_pd(func(1), func(0));
+    }
+
+    NETGEN_INLINE double operator[] (int i) const { return ((double*)(&data))[i]; }
+    NETGEN_INLINE __m128d Data() const { return data; }
+    NETGEN_INLINE __m128d & Data() { return data; }
+
+    template <int I>
+    double Get() const
+    {
+      static_assert(I>=0 && I<2, "Index out of range");
+      return (*this)[I];
+    }
+
+    double Lo() const { return Get<0>(); } 
+    double Hi() const { return Get<1>(); } 
+    
+    operator std::tuple<double&,double&> ()
+    {
+      auto pdata = (double*)&data;
+      return std::tuple<double&,double&>(pdata[0], pdata[1]);
+    }
+  };
+
+  NETGEN_INLINE SIMD<double,2> operator- (SIMD<double,2> a) { return _mm_xor_pd(a.Data(), _mm_set1_pd(-0.0)); }
+  NETGEN_INLINE SIMD<double,2> operator+ (SIMD<double,2> a, SIMD<double,2> b) { return _mm_add_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,2> operator- (SIMD<double,2> a, SIMD<double,2> b) { return _mm_sub_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,2> operator* (SIMD<double,2> a, SIMD<double,2> b) { return _mm_mul_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,2> operator/ (SIMD<double,2> a, SIMD<double,2> b) { return _mm_div_pd(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<double,2> operator* (double a, SIMD<double,2> b) { return _mm_set1_pd(a)*b; }
+  NETGEN_INLINE SIMD<double,2> operator* (SIMD<double,2> b, double a) { return _mm_set1_pd(a)*b; }
+
+  template<>
+  NETGEN_INLINE auto Unpack (SIMD<double,2> a, SIMD<double,2> b)
+  {
+    return std::make_tuple(SIMD<double,2>(_mm_unpacklo_pd(a.Data(),b.Data())),
+                      SIMD<double,2>(_mm_unpackhi_pd(a.Data(),b.Data())));
+  }
+
+  NETGEN_INLINE __m128d my_mm_hadd_pd(__m128d a, __m128d b) {
+#if defined(__SSE3__) || defined(__AVX__)
+    return _mm_hadd_pd(a,b);
+#else
+    return _mm_add_pd( _mm_unpacklo_pd(a,b), _mm_unpackhi_pd(a,b) );
+#endif
+  }
+
+#ifndef __AVX__
+  NETGEN_INLINE __m128i my_mm_cmpgt_epi64(__m128i a, __m128i b) {
+    auto  res_lo = _mm_cvtsi128_si64(a)  > _mm_cvtsi128_si64(b) ? -1:0;
+    auto  res_hi = _mm_cvtsi128_si64(_mm_srli_si128(a,8)) > _mm_cvtsi128_si64(_mm_srli_si128(b,8)) ? -1 : 0;
+    return _mm_set_epi64x(res_hi,res_lo);
+  }
+#else
+  NETGEN_INLINE __m128i my_mm_cmpgt_epi64(__m128i a, __m128i b) {
+    return _mm_cmpgt_epi64(a,b);
+  }
+#endif
+
+
+  NETGEN_INLINE SIMD<double,2> sqrt (SIMD<double,2> a) { return _mm_sqrt_pd(a.Data()); }
+  NETGEN_INLINE SIMD<double,2> fabs (SIMD<double,2> a) { return _mm_max_pd(a.Data(), (-a).Data()); }
+  using std::floor;
+  NETGEN_INLINE SIMD<double,2> floor (SIMD<double,2> a)
+  { return ngcore::SIMD<double,2>([&](int i)->double { return floor(a[i]); } ); }
+  using std::ceil;
+  NETGEN_INLINE SIMD<double,2> ceil (SIMD<double,2> a)
+  { return ngcore::SIMD<double,2>([&](int i)->double { return ceil(a[i]); } ); }
+
+  NETGEN_INLINE SIMD<mask64,2> operator<= (SIMD<double,2> a , SIMD<double,2> b)
+  { return _mm_castpd_si128( _mm_cmple_pd(a.Data(),b.Data())); }
+  NETGEN_INLINE SIMD<mask64,2> operator< (SIMD<double,2> a , SIMD<double,2> b)
+  { return _mm_castpd_si128( _mm_cmplt_pd(a.Data(),b.Data())); }
+  NETGEN_INLINE SIMD<mask64,2> operator>= (SIMD<double,2> a , SIMD<double,2> b)
+  { return _mm_castpd_si128( _mm_cmpge_pd(a.Data(),b.Data())); }
+  NETGEN_INLINE SIMD<mask64,2> operator> (SIMD<double,2> a , SIMD<double,2> b)
+  { return _mm_castpd_si128( _mm_cmpgt_pd(a.Data(),b.Data())); }
+  NETGEN_INLINE SIMD<mask64,2> operator== (SIMD<double,2> a , SIMD<double,2> b)
+  { return _mm_castpd_si128( _mm_cmpeq_pd(a.Data(),b.Data())); }
+  NETGEN_INLINE SIMD<mask64,2> operator!= (SIMD<double,2> a , SIMD<double,2> b)
+  { return _mm_castpd_si128( _mm_cmpneq_pd(a.Data(),b.Data())); }
+
+  NETGEN_INLINE SIMD<mask64,2> operator<= (SIMD<int64_t,2> a , SIMD<int64_t,2> b)
+  { return  _mm_xor_si128(_mm_cmpgt_epi64(a.Data(),b.Data()),_mm_set1_epi32(-1)); }
+  NETGEN_INLINE SIMD<mask64,2> operator< (SIMD<int64_t,2> a , SIMD<int64_t,2> b)
+  { return  my_mm_cmpgt_epi64(b.Data(),a.Data()); }
+  NETGEN_INLINE SIMD<mask64,2> operator>= (SIMD<int64_t,2> a , SIMD<int64_t,2> b)
+  { return  _mm_xor_si128(_mm_cmpgt_epi64(b.Data(),a.Data()),_mm_set1_epi32(-1)); }
+  NETGEN_INLINE SIMD<mask64,2> operator> (SIMD<int64_t,2> a , SIMD<int64_t,2> b)
+  { return  my_mm_cmpgt_epi64(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<mask64,2> operator== (SIMD<int64_t,2> a , SIMD<int64_t,2> b)
+  { return  _mm_cmpeq_epi64(a.Data(),b.Data()); }
+  NETGEN_INLINE SIMD<mask64,2> operator!= (SIMD<int64_t,2> a , SIMD<int64_t,2> b)
+  { return  _mm_xor_si128(_mm_cmpeq_epi64(a.Data(),b.Data()),_mm_set1_epi32(-1)); }
+
+
+
+ NETGEN_INLINE SIMD<mask64,2> operator&& (SIMD<mask64,2> a, SIMD<mask64,2> b)
+  { return _mm_castpd_si128(_mm_and_pd (_mm_castsi128_pd(a.Data()),_mm_castsi128_pd( b.Data()))); }
+  NETGEN_INLINE SIMD<mask64,2> operator|| (SIMD<mask64,2> a, SIMD<mask64,2> b)
+  { return _mm_castpd_si128(_mm_or_pd (_mm_castsi128_pd(a.Data()), _mm_castsi128_pd(b.Data()))); }
+  NETGEN_INLINE SIMD<mask64,2> operator! (SIMD<mask64,2> a)
+  { return _mm_castpd_si128(_mm_xor_pd (_mm_castsi128_pd(a.Data()),_mm_castsi128_pd( _mm_cmpeq_epi64(a.Data(),a.Data())))); }
+#ifdef __SSE4_1__
+  NETGEN_INLINE SIMD<double,2> If (SIMD<mask64,2> a, SIMD<double,2> b, SIMD<double,2> c)
+  { return _mm_blendv_pd(c.Data(), b.Data(), _mm_castsi128_pd(a.Data())); }
+#else
+  NETGEN_INLINE SIMD<double,2> If (SIMD<mask64,2> a, SIMD<double,2> b, SIMD<double,2> c)
+  {
+    return _mm_or_pd(
+                      _mm_andnot_pd(_mm_castsi128_pd(a.Data()),c.Data()),
+                      _mm_and_pd(b.Data(),_mm_castsi128_pd(a.Data()))
+                      );}
+#endif // __SSE4_1__
+
+  NETGEN_INLINE SIMD<double,2> IfPos (SIMD<double,2> a, SIMD<double,2> b, SIMD<double,2> c)
+  { return ngcore::SIMD<double,2>([&](int i)->double { return a[i]>0 ? b[i] : c[i]; }); }
+  NETGEN_INLINE SIMD<double,2> IfZero (SIMD<double,2> a, SIMD<double,2> b, SIMD<double,2> c)
+  { return ngcore::SIMD<double,2>([&](int i)->double { return a[i]==0. ? b[i] : c[i]; }); }
+
+
+  NETGEN_INLINE double HSum (SIMD<double,2> sd)
+  {
+    return _mm_cvtsd_f64 (my_mm_hadd_pd (sd.Data(), sd.Data()));
+  }
+
+  NETGEN_INLINE auto HSum (SIMD<double,2> sd1, SIMD<double,2> sd2)
+  {
+    __m128d hv2 = my_mm_hadd_pd(sd1.Data(), sd2.Data());
+    return SIMD<double,2> (hv2);
+    // return SIMD<double,2>(_mm_cvtsd_f64 (hv2),  _mm_cvtsd_f64(_mm_shuffle_pd (hv2, hv2, 3)));
+  }
+
+  NETGEN_INLINE SIMD<int64_t, 2> If(SIMD<mask64, 2> a, SIMD<int64_t, 2> b,
+                             SIMD<int64_t, 2> c) {
+    return _mm_or_si128(
+                        _mm_andnot_si128(a.Data(),c.Data()),
+                        _mm_and_si128(b.Data(),a.Data())
+                        );
+  }
+
+}
+
+#endif // NETGEN_CORE_SIMD_SSE_HPP

libsrc/core/symboltable.hpp

@@ -5,7 +5,6 @@
 #include <string>
 #include <vector>
 
-#include "archive.hpp"
 #include "exception.hpp"
 #include "ngcore_api.hpp"
 
@@ -38,14 +37,15 @@ namespace ngcore
     SymbolTable& operator=(const SymbolTable<T>&) = default;
     SymbolTable& operator=(SymbolTable<T>&&) = default;
 
-    template<typename T2=T>
-    auto DoArchive(Archive& ar) -> typename std::enable_if_t<is_archivable<T2>>
+    template<typename ARCHIVE>
+    auto DoArchive(ARCHIVE& ar)
+      -> typename std::enable_if_t<ARCHIVE::template is_archivable<T>, void>
     {
       ar & names & data;
     }
 
     /// INDEX of symbol name, throws exception if unused
-    size_t Index (const std::string & name) const
+    size_t Index (std::string_view name) const
     {
       for (size_t i = 0; i < names.size(); i++)
         if (names[i] == name) return i;
@@ -53,7 +53,7 @@ namespace ngcore
     }
 
     /// Index of symbol name, returns -1 if unused
-    int CheckIndex (const std::string & name) const
+    int CheckIndex (std::string_view name) const
     {
       for (int i = 0; i < names.size(); i++)
         if (names[i] == name) return i;
@@ -67,12 +67,12 @@ namespace ngcore
     }
 
     /// Returns reference to element. exception for unused identifier
-    reference operator[] (const std::string & name)
+    reference operator[] (std::string_view name)
     {
       return data[Index (name)];
     }
 
-    const_reference operator[] (const std::string & name) const
+    const_reference operator[] (std::string_view name) const
     {
       return data[Index (name)];
     }
@@ -99,7 +99,7 @@ namespace ngcore
     }
 
     /// Associates el to the string name, overrides if name is used
-    void Set (const std::string & name, const T & el)
+    void Set (std::string_view name, const T & el)
     {
       int i = CheckIndex (name);
       if (i >= 0)
@@ -107,15 +107,24 @@ namespace ngcore
       else
         {
           data.push_back(el);
-          names.push_back(name);
+          names.push_back(std::string(name));
         }
     }
 
+
+    
+    /*
     bool Used (const std::string & name) const
     {
       return CheckIndex(name) >= 0;
     }
-
+    */
+    
+    bool Used (std::string_view name) const
+    {
+      return CheckIndex(name) >= 0;
+    }
+    
     /// Deletes symboltable
     inline void DeleteAll ()
     {

libsrc/core/table.cpp

@@ -18,6 +18,19 @@ namespace ngcore
     size_t size  = entrysize.Size();
     size_t * index = new size_t[size+1];
 
+    if (entrysize.Size() < 100)
+      {
+        size_t mysum = 0;
+        for (size_t i = 0; i < entrysize.Size(); i++)
+          {
+            index[i] = mysum;
+            mysum += entrysize[i];
+          }
+        index[entrysize.Size()] = mysum;
+        return index;
+      }
+
+    
     Array<size_t> partial_sums(TaskManager::GetNumThreads()+1);
     partial_sums[0] = 0;
     ParallelJob
@@ -49,12 +62,12 @@ namespace ngcore
     return index;
   }
 
-  NGCORE_API size_t * TablePrefixSum32 (FlatArray<unsigned int> entrysize)
+  NGCORE_API size_t * TablePrefixSum32 (FlatArray<uint32_t> entrysize)
   { return TablePrefixSum2 (entrysize); }
-  NGCORE_API size_t * TablePrefixSum64 (FlatArray<size_t> entrysize)
+  NGCORE_API size_t * TablePrefixSum64 (FlatArray<uint64_t> entrysize)
   { return TablePrefixSum2 (entrysize); }
 
-
+  /*
   BaseDynamicTable :: BaseDynamicTable (int size)
     : data(size)
   {
@@ -88,7 +101,6 @@ namespace ngcore
       }
   }
 
-
   BaseDynamicTable :: ~BaseDynamicTable ()
   {
     if (oneblock)
@@ -112,7 +124,7 @@ namespace ngcore
       }
   }
 
-  void BaseDynamicTable :: IncSize (int i, int elsize)
+  void BaseDynamicTable :: IncSize (IndexType i, int elsize)
   {
     if (i < 0 || i >= data.Size())
       {
@@ -135,7 +147,7 @@ namespace ngcore
     line.size++;
   }
 
-  void BaseDynamicTable :: DecSize (int i)
+  void BaseDynamicTable :: DecSize (IndexType i)
   {
     if (i < 0 || i >= data.Size())
       {
@@ -153,6 +165,7 @@ namespace ngcore
 
     line.size--;
   }
+  */
 
   void FilteredTableCreator::Add (size_t blocknr, int data)
   {

libsrc/core/table.hpp

@@ -9,88 +9,92 @@
 
 #include <atomic>
 #include <iostream>
+#include <optional>
 
 #include "array.hpp"
 #include "bitarray.hpp"
-#include "taskmanager.hpp"
+#include "memtracer.hpp"
 #include "ngcore_api.hpp"
+#include "profiler.hpp"
+
 
 namespace ngcore
 {
 
 
   template <class T, class IndexType = size_t>
-class FlatTable
-{
-protected:
-  static constexpr IndexType BASE = IndexBASE<IndexType>();  
-  /// number of rows
-  size_t size;
-  /// pointer to first in row
-  size_t * index;
-  /// array of data
-  T * data;
-
-public:
-  FlatTable() = delete;
-
-  NETGEN_INLINE FlatTable(size_t as, size_t * aindex, T * adata)
-    : size(as), index(aindex), data(adata) { ; }
-
-  /// Size of table
-  NETGEN_INLINE size_t Size() const { return size; }
-
-  /// Access entry
-  NETGEN_INLINE const FlatArray<T> operator[] (IndexType i) const
+  class FlatTable
   {
-    i = i-BASE;
-    return FlatArray<T> (index[i+1]-index[i], data+index[i]);
-  }
+  protected:
+    static constexpr IndexType BASE = IndexBASE<IndexType>();  
+    /// number of rows
+    size_t size;
+    /// pointer to first in row
+    size_t * index;
+    /// array of data
+    T * data;
 
-  NETGEN_INLINE T * Data() const { return data; }
-
-  NETGEN_INLINE FlatArray<T> AsArray() const
-  {
-    return FlatArray<T> (index[size]-index[0], data+index[0]);
-  }
-
-  NETGEN_INLINE FlatArray<size_t> IndexArray() const
-  {
-    return FlatArray<size_t, IndexType> (size+1, index);
-  }
-
-  /// takes range starting from position start of end-start elements
-  NETGEN_INLINE FlatTable<T> Range (size_t start, size_t end) const
-  {
-    return FlatTable<T> (end-start, index+start-BASE, data);
-  }
-
-  /// takes range starting from position start of end-start elements
-  NETGEN_INLINE FlatTable<T> Range (T_Range<size_t> range) const
-  {
-    return FlatTable<T> (range.Size(), index+range.First()-BASE, data);
-  }
-
-  NETGEN_INLINE T_Range<IndexType> Range () const
-  {
-    return T_Range<IndexType> (BASE, size+BASE);
-  }
-  
-  class Iterator
-  {
-    const FlatTable & tab;
-    size_t row;
   public:
-    Iterator (const FlatTable & _tab, size_t _row) : tab(_tab), row(_row) { ; }
-    Iterator & operator++ () { ++row; return *this; }
-    FlatArray<T> operator* () const { return tab[row]; }
-    bool operator!= (const Iterator & it2) { return row != it2.row; }
+    FlatTable() = delete;
+    FlatTable (const FlatTable &) = default;
+    
+    NETGEN_INLINE FlatTable(size_t as, size_t * aindex, T * adata)
+      : size(as), index(aindex), data(adata) { ; }
+
+    /// Size of table
+    NETGEN_INLINE size_t Size() const { return size; }
+
+    /// Access entry
+    NETGEN_INLINE const FlatArray<T> operator[] (IndexType i) const
+    {
+      return FlatArray<T> (index[i-BASE+1]-index[i-BASE], data+index[i-BASE]);
+    }
+
+    NETGEN_INLINE T * Data() const { return data; }
+
+    NETGEN_INLINE FlatArray<T> AsArray() const
+    {
+      return FlatArray<T> (index[size]-index[0], data+index[0]);
+    }
+
+    NETGEN_INLINE FlatArray<size_t> IndexArray() const
+    {
+      return FlatArray<size_t, IndexType> (size+1, index);
+    }
+
+    /// takes range starting from position start of end-start elements
+    NETGEN_INLINE FlatTable<T> Range (size_t start, size_t end) const
+    {
+      return FlatTable<T> (end-start, index+start-BASE, data);
+    }
+
+    /// takes range starting from position start of end-start elements
+    NETGEN_INLINE FlatTable<T> Range (T_Range<size_t> range) const
+    {
+      return FlatTable<T> (range.Size(), index+range.First()-BASE, data);
+    }
+
+    NETGEN_INLINE T_Range<IndexType> Range () const
+    {
+      return T_Range<IndexType> (BASE, size+BASE);
+    }
+  
+    class Iterator
+    {
+      const FlatTable & tab;
+      size_t row;
+    public:
+      Iterator (const FlatTable & _tab, size_t _row) : tab(_tab), row(_row) { ; }
+      Iterator & operator++ () { ++row; return *this; }
+      FlatArray<T> operator* () const { return tab[row]; }
+      bool operator!= (const Iterator & it2) { return row != it2.row; }
+    };
+
+    Iterator begin() const { return Iterator(*this, BASE); }
+    Iterator end() const { return Iterator(*this, BASE+size); }
   };
 
-  Iterator begin() const { return Iterator(*this, BASE); }
-  Iterator end() const { return Iterator(*this, BASE+size); }
-};
-
+  /*
   NGCORE_API extern size_t * TablePrefixSum32 (FlatArray<unsigned int> entrysize);
   NGCORE_API extern size_t * TablePrefixSum64 (FlatArray<size_t> entrysize);
 
@@ -103,108 +107,180 @@ public:
   { return TablePrefixSum32 (FlatArray<unsigned> (entrysize.Size(), (unsigned int*)(std::atomic<int>*)entrysize.Addr(0))); }
   NETGEN_INLINE size_t * TablePrefixSum (FlatArray<size_t> entrysize)
   { return TablePrefixSum64 (entrysize); }
+  */
 
+  NGCORE_API extern size_t * TablePrefixSum32 (FlatArray<uint32_t> entrysize);
+  NGCORE_API extern size_t * TablePrefixSum64 (FlatArray<uint64_t> entrysize);
 
-/**
-    A compact Table container.
-    A table contains size entries of variable size.
-    The entry sizes must be known at construction.
-*/
+  template <typename T> // TODO: enable_if T is integral
+  NETGEN_INLINE size_t * TablePrefixSum (FlatArray<T> entrysize)
+  {
+    if constexpr (sizeof(T) == 4)
+      return TablePrefixSum32 ( { entrysize.Size(), (uint32_t*)(void*)entrysize.Addr(0) });
+    else
+      return TablePrefixSum64 ( { entrysize.Size(), (uint64_t*)(void*)entrysize.Addr(0) });
+  }
+  
+
+  /**
+     A compact Table container.
+     A table contains size entries of variable size.
+     The entry sizes must be known at construction.
+  */
   template <class T, class IndexType = size_t>
   class Table : public FlatTable<T, IndexType>
-{
-protected:
-
-  using FlatTable<T,IndexType>::size;
-  using FlatTable<T,IndexType>::index;
-  using FlatTable<T,IndexType>::data;
-
-public:
-  ///
-  NETGEN_INLINE Table () : FlatTable<T,IndexType> (0,nullptr,nullptr) { ; }
-  /// Construct table of uniform entrysize
-  NETGEN_INLINE Table (size_t asize, size_t entrysize)
-    : FlatTable<T,IndexType>( asize, new size_t[asize+1], new T[asize*entrysize] )
   {
-    for (size_t i : IntRange(size+1))
-      index[i] = i*entrysize;
-  }
+  protected:
 
-  /// Construct table of variable entrysize
-  template <typename TI>
-  NETGEN_INLINE Table (FlatArray<TI,IndexType> entrysize)
-    : FlatTable<T,IndexType> (0, nullptr, nullptr)
-  {
-    size  = entrysize.Size();
-    index = TablePrefixSum (FlatArray<TI> (entrysize.Size(), entrysize.Data()));
-    size_t cnt = index[size];
-    data = new T[cnt];
-  }
+    using FlatTable<T,IndexType>::size;
+    using FlatTable<T,IndexType>::index;
+    using FlatTable<T,IndexType>::data;
 
-  explicit NETGEN_INLINE Table (const Table & tab2)
-    : FlatTable<T,IndexType>(0, nullptr, nullptr)
-  {
-    size = tab2.Size();
+  public:
+    ///
+    NETGEN_INLINE Table () : FlatTable<T,IndexType> (0,nullptr,nullptr) { ; }
+    /// Construct table of uniform entrysize
+    NETGEN_INLINE Table (size_t asize, size_t entrysize)
+      : FlatTable<T,IndexType>( asize, new size_t[asize+1], new T[asize*entrysize] )
+    {
+      for (size_t i : IntRange(size+1))
+        index[i] = i*entrysize;
+      mt.Alloc(GetMemUsage());
+    }
 
-    index = new size_t[size+1];
-    for (size_t i = 0; i <= size; i++)
-      index[i] = tab2.index[i];
+    /// Construct table of variable entrysize
+    template <typename TI>
+    NETGEN_INLINE Table (FlatArray<TI,IndexType> entrysize)
+      : FlatTable<T,IndexType> (0, nullptr, nullptr)
+    {
+      size  = entrysize.Size();
+      index = TablePrefixSum (FlatArray<TI> (entrysize.Size(), entrysize.Data()));
+      size_t cnt = index[size];
+      data = new T[cnt];
+      mt.Alloc(GetMemUsage());
+    }
 
-    size_t cnt = index[size];
-    data = new T[cnt];
-    for (size_t i = 0; i < cnt; i++)
-      data[i] = tab2.data[i];
-  }
+    explicit NETGEN_INLINE Table (const FlatTable<T,IndexType> & tab2)
+      : FlatTable<T,IndexType>(0, nullptr, nullptr)
+    {
+      size = tab2.Size();
+      if (size == 0) return;
+      
+      index = new size_t[size+1];
+      this->IndexArray() = tab2.IndexArray();
+      // for (size_t i = 0; i <= size; i++)
+      // index[i] = tab2.index[i];
 
-  NETGEN_INLINE Table (Table && tab2)
-    : FlatTable<T,IndexType>(0, nullptr, nullptr)
-  {
-    Swap (size, tab2.size);
-    Swap (index, tab2.index);
-    Swap (data, tab2.data);
-  }
+      size_t cnt = index[size];
+      data = new T[cnt];
+      this->AsArray() = tab2.AsArray();
+      mt.Alloc(GetMemUsage());
+      /*
+      for (size_t i = 0; i < cnt; i++)
+        data[i] = tab2.data[i];
+      */
+    }
+    
+    explicit NETGEN_INLINE Table (const Table & tab2)
+      : FlatTable<T,IndexType>(0, nullptr, nullptr)
+    {
+      size = tab2.Size();
+      if (size == 0) return;
+      
+      index = new size_t[size+1];
+      for (size_t i = 0; i <= size; i++)
+        index[i] = tab2.index[i];
 
-  NETGEN_INLINE Table & operator= (Table && tab2)
-  {
-    Swap (size, tab2.size);
-    Swap (index, tab2.index);
-    Swap (data, tab2.data);
-    return *this;
-  }
+      size_t cnt = index[size];
+      data = new T[cnt];
+      for (size_t i = 0; i < cnt; i++)
+        data[i] = tab2.data[i];
+
+      mt.Alloc(GetMemUsage());
+    }
+
+    NETGEN_INLINE Table (Table && tab2)
+      : FlatTable<T,IndexType>(0, nullptr, nullptr)
+    {
+      mt = std::move(tab2.mt);
+      Swap (size, tab2.size);
+      Swap (index, tab2.index);
+      Swap (data, tab2.data);
+    }
+
+    template<typename ARCHIVE>
+    auto DoArchive(ARCHIVE& ar)
+    {
+      ar & size;
+      if(size == 0)
+        return;
+      if(ar.Input())
+        {
+          index = new IndexType[size+1];
+          mt.Alloc(sizeof(IndexType) * (size+1));
+        }
+      ar.Do(index, size+1);
+      if(ar.Input())
+        {
+          data = new T[index[size]];
+          mt.Alloc(sizeof(T) * index[size]);
+        }
+      ar.Do(data, index[size]);
+    }
+
+    NETGEN_INLINE Table & operator= (Table && tab2)
+    {
+      mt = std::move(tab2.mt);
+      Swap (size, tab2.size);
+      Swap (index, tab2.index);
+      Swap (data, tab2.data);
+      return *this;
+    }
 
 
 
-  /// Delete data
-  NETGEN_INLINE ~Table ()
-  {
-    delete [] data;
-    delete [] index;
-  }
+    /// Delete data
+    NETGEN_INLINE ~Table ()
+    {
+      mt.Free(GetMemUsage());
+      delete [] data;
+      delete [] index;
+    }
 
-  /// Size of table
-  using FlatTable<T,IndexType>::Size;
+    /// Size of table
+    using FlatTable<T,IndexType>::Size;
 
-  /// number of elements in all rows
-  NETGEN_INLINE size_t NElements() const { return index[size]; }
+    /// number of elements in all rows
+    NETGEN_INLINE size_t NElements() const { return index[size]; }
 
-  using FlatTable<T,IndexType>::operator[];
-};
+    using FlatTable<T,IndexType>::operator[];
+
+    NETGEN_INLINE void StartMemoryTracing (int /* mem_id */)
+    {
+      mt.Alloc(GetMemUsage());
+    }
+    const MemoryTracer& GetMemoryTracer() const { return mt; }
+
+  private:
+    size_t GetMemUsage() const { return size == 0 ? 0 : sizeof(T)*index[size] + sizeof(IndexType) * size+1; }
+    MemoryTracer mt;
+  };
 
 
-/// Print table
+  /// Print table
   template <class T, typename IndexType>
   inline ostream & operator<< (ostream & s, const Table<T,IndexType> & table)
-{
-  for (auto i : table.Range())
-    {
-      s << i << ":";
-      for (auto el : table[i])
-	s << " " << el;
-      s << "\n";
-    }
-  s << std::flush;
-  return s;
-}
+  {
+    for (auto i : table.Range())
+      {
+        s << i << ":";
+        for (auto el : table[i])
+          s << " " << el;
+        s << "\n";
+      }
+    s << std::flush;
+    return s;
+  }
 
 
 
@@ -268,8 +344,8 @@ public:
 	case 1:
           {
             size_t oldval = nd;
-            while (blocknr+1>nd) {
-              nd.compare_exchange_weak (oldval, blocknr+1);
+            while (blocknr-IndexBASE<IndexType>()+1>nd) {
+              nd.compare_exchange_weak (oldval, blocknr-IndexBASE<IndexType>()+1);
               oldval = nd;
             }
             break;
@@ -334,6 +410,49 @@ public:
     }
   };
 
+  template <typename TEntry, typename TIndex, typename TRange, typename TFunc>
+  Table<TEntry, TIndex> CreateTable( const TRange & range, const TFunc & func, std::optional< size_t > cnt )
+  {
+      static Timer timer("CreateTable");
+      RegionTimer rt(timer);
+      std::unique_ptr<TableCreator<TEntry, TIndex>> pcreator;
+
+      if(cnt)
+          pcreator = std::make_unique<TableCreator<TEntry, TIndex>>(*cnt);
+      else
+          pcreator = std::make_unique<TableCreator<TEntry, TIndex>>();
+      
+      auto & creator = *pcreator;
+
+      for ( ; !creator.Done(); creator++)
+        ParallelForRange
+          (range, [&] (auto myrange)
+           {
+             for (auto i : myrange)
+               func(creator, i);
+           }, TasksPerThread(4)
+          );
+
+    return creator.MoveTable();
+  }
+
+  template <typename TEntry, typename TIndex, typename TRange, typename TFunc>
+  Table<TEntry, TIndex> CreateSortedTable( const TRange & range, const TFunc & func, std::optional< size_t > cnt )
+  {
+    static Timer timer("CreateSortedTable");
+    RegionTimer rt(timer);
+    Table<TEntry, TIndex> table = CreateTable<TEntry, TIndex>(range, func, cnt);
+    ParallelForRange
+      (table.Range(), [&] (auto myrange)
+       {
+         for (auto i : myrange)
+           QuickSort(table[i]);
+       }, TasksPerThread(4)
+      );
+
+    return table;
+  }
+
   class NGCORE_API FilteredTableCreator : public TableCreator<int>
   {
   protected:
@@ -348,143 +467,268 @@ public:
     void Add (size_t blocknr, FlatArray<int> dofs);
   };
 
+  
 
-  /// Base class to generic DynamicTable.
-  class BaseDynamicTable
-  {
-  protected:
-
-    ///
-    struct linestruct
-    {
-      ///
-      int size;
-      ///
-      int maxsize;
-      ///
-      void * col;
-    };
-
-    ///
-    Array<linestruct> data;
-    ///
-    char * oneblock;
-
-  public:
-    ///
-    NGCORE_API BaseDynamicTable (int size);
-    ///
-    NGCORE_API BaseDynamicTable (const Array<int> & entrysizes, int elemsize);
-    ///
-    NGCORE_API ~BaseDynamicTable ();
-
-    /// Changes Size of table to size, deletes data
-    NGCORE_API void SetSize (int size);
-    ///
-    NGCORE_API void IncSize (int i, int elsize);
-
-    NGCORE_API void DecSize (int i);
-  };
-
-
+  
 
   /**
-      A dynamic table class.
+     A dynamic table class.
 
-      A DynamicTable contains entries of variable size. Entry sizes can
-      be increased dynamically.
+     A DynamicTable contains entries of variable size. Entry sizes can
+     be increased dynamically.
   */
-  template <class T>
-  class DynamicTable : public BaseDynamicTable
+  template <class T, class IndexType = size_t>
+  class DynamicTable 
   {
+  protected:
+    static constexpr IndexType BASE = IndexBASE<IndexType>();
+
+    struct linestruct
+    {
+      int size;
+      int maxsize;
+      T * col;
+    };
+
+    Array<linestruct, IndexType> data;
+    T * oneblock = nullptr;
+  
   public:
     /// Creates table of size size
     DynamicTable (int size = 0)
-      : BaseDynamicTable (size) { ; }
-
-    /// Creates table with a priori fixed entry sizes.
-    DynamicTable (const Array<int> & entrysizes)
-      : BaseDynamicTable (entrysizes, sizeof(T)) { ; }
-
-    /// Inserts element acont into row i. Does not test if already used.
-    void Add (int i, const T & acont)
+      : data(size)
     {
-      if (data[i].size == data[i].maxsize)
-        IncSize (i, sizeof (T));
-      else
-        data[i].size++;
-      static_cast<T*> (data[i].col) [data[i].size-1] = acont;
+      for (auto & d : data)
+        {
+          d.maxsize = 0;
+          d.size = 0;
+          d.col = nullptr;
+        }
+      oneblock = nullptr;
     }
 
+    /// Creates table with a priori fixed entry sizes.
+    DynamicTable (const Array<int, IndexType> & entrysizes, bool setentrysize=false)
+      : data(entrysizes.Size())
+    {
+      size_t cnt = 0;
+      // size_t n = entrysizes.Size();
+      
+      for (auto es : entrysizes)
+        cnt += es;
+      oneblock = new T[cnt];
+      
+      cnt = 0;
+      for (auto i : data.Range())
+        {
+          data[i].maxsize = entrysizes[i];
+          if (setentrysize)
+            data[i].size = entrysizes[i];
+          else
+            data[i].size = 0;
+          data[i].col = &oneblock[cnt];
+          cnt += entrysizes[i];
+        }
+    }
+
+    DynamicTable (DynamicTable && tab2)
+    {
+      Swap (data, tab2.data);
+      Swap (oneblock, tab2.oneblock);
+    }
+
+    ~DynamicTable ()
+    {
+      if (oneblock)
+        delete [] oneblock;
+      else
+        for (auto & d : data)
+          delete [] d.col;
+    }
+    
+    DynamicTable & operator= (DynamicTable && tab2)
+    {
+      Swap (data, tab2.data);
+      Swap (oneblock, tab2.oneblock);
+      return *this;
+    }
+
+    /// Changes Size of table to size, deletes data
+    void SetSize (int size)
+    {
+      for (auto & d : data)
+        delete [] d.col;
+    
+      data.SetSize(size);
+      for (auto & d : data)
+        {
+          d.maxsize = 0;
+          d.size = 0;
+          d.col = nullptr;
+        }
+    }      
+
+    void ChangeSize (size_t size)
+    {
+      if (oneblock)
+        throw Exception ("cannot change size of oneblock dynamic table");
+      
+      size_t oldsize = data.Size();
+      if (size == oldsize) 
+        return;
+      
+      if (size < oldsize)
+        for (int i = size; i < oldsize; i++)
+          delete [] data[i+BASE].col;
+      
+      data.SetSize(size);
+
+      for (int i = oldsize; i < size; i++)
+        {
+          data[i+BASE].maxsize = 0;
+          data[i+BASE].size = 0;
+          data[i+BASE].col = nullptr;
+        }    
+    }
+    
+
+    
+    ///
+    void IncSize (IndexType i)
+    {
+      NETGEN_CHECK_RANGE(i,BASE,data.Size()+BASE);
+    
+      linestruct & line = data[i];
+    
+      if (line.size == line.maxsize)
+        {
+          T * p;
+          if constexpr (std::is_default_constructible<T>::value)
+            p = new T[(2*line.maxsize+5)];
+          else
+            p = reinterpret_cast<T*>(new char[(2*line.maxsize+5)*sizeof(T)]);
+          for (size_t i = 0; i < line.maxsize; i++)
+            p[i] = std::move(line.col[i]);
+          // memcpy (p, line.col, line.maxsize * sizeof(T));
+          delete [] line.col;
+          line.col = p;
+          line.maxsize = 2*line.maxsize+5;
+        }
+    
+      line.size++;
+    }
+  
+    void DecSize (IndexType i)
+    {
+      NETGEN_CHECK_RANGE(i,BASE,data.Size()+BASE);
+      linestruct & line = data[i];
+    
+#ifdef NETGEN_ENABLE_CHECK_RANGE
+      if (line.size == 0)
+        throw Exception ("BaseDynamicTable::Dec: EntrySize < 0");
+#endif
+    
+      line.size--;
+    }
+  
+  
+    /// Inserts element acont into row i. Does not test if already used.
+    void Add (IndexType i, const T & acont)
+    {
+      if (data[i].size == data[i].maxsize)
+        this->IncSize (i);
+      else
+        data[i].size++;
+      data[i].col[data[i].size-1] = acont;
+    }
+  
     /// Inserts element acont into row i, iff not yet exists.
-    void AddUnique (int i, const T & cont)
+    void AddUnique (IndexType i, const T & cont)
     {
       int es = EntrySize (i);
-      int * line = const_cast<int*> (GetLine (i));
+      T * line = data[i].col;
       for (int j = 0; j < es; j++)
         if (line[j] == cont)
           return;
       Add (i, cont);
     }
-
+  
 
     /// Inserts element acont into row i. Does not test if already used.
-    void AddEmpty (int i)
+    void AddEmpty (IndexType i)
     {
-      IncSize (i, sizeof (T));
+      IncSize (i);
     }
-
+  
     /** Set the nr-th element in the i-th row to acont.
         Does not check for overflow. */
-    void Set (int i, int nr, const T & acont)
-    { static_cast<T*> (data[i].col)[nr] = acont; }
+    void Set (IndexType i, int nr, const T & acont)
+    {
+      data[i].col[nr] = acont;
+    }
 
 
     /** Returns the nr-th element in the i-th row.
-      Does not check for overflow. */
-    const T & Get (int i, int nr) const
-    { return static_cast<T*> (data[i].col)[nr]; }
-
-
+        Does not check for overflow. */
+    const T & Get (IndexType i, int nr) const
+    {
+      return data[i].col[nr];
+    }
+  
+  
     /** Returns pointer to the first element in row i. */
-    const T * GetLine (int i) const
-    { return static_cast<T*> (data[i].col); }
-
-
+    const T * GetLine (IndexType i) const
+    {
+      return data[i].col;
+    }
+  
     /// Returns size of the table.
-    int Size () const
-    { return data.Size(); }
+    size_t Size () const
+    {
+      return data.Size();
+    }
+
+    auto Range () const
+    {
+      return data.Range();
+    }
 
     /// Returns size of the i-th row.
-    int EntrySize (int i) const
-    { return data[i].size; }
+    int EntrySize (IndexType i) const
+    {
+      return data[i].size;
+    }
 
     ///
-    void DecEntrySize (int i)
-    { DecSize(i); }
-
+    void DecEntrySize (IndexType i)
+    {
+      DecSize(i);
+    }
+  
     /// Access entry i
-    FlatArray<T> operator[] (int i)
-    { return FlatArray<T> (data[i].size, static_cast<T*> (data[i].col)); }
-
+    FlatArray<T> operator[] (IndexType i)
+    {
+      return FlatArray<T> (data[i].size, data[i].col);
+    }
+  
     /*
-    typedef const FlatArray<T> ConstFlatArray;
-    /// Access entry i
-    ConstFlatArray operator[] (int i) const
-    { return FlatArray<T> (data[i].size, static_cast<T*> (data[i].col)); }
+      typedef const FlatArray<T> ConstFlatArray;
+      /// Access entry i
+      ConstFlatArray operator[] (int i) const
+      { return FlatArray<T> (data[i].size, static_cast<T*> (data[i].col)); }
     */
-    FlatArray<T> operator[] (int i) const
-    { return FlatArray<T> (data[i].size, static_cast<T*> (data[i].col)); }
+    FlatArray<T> operator[] (IndexType i) const
+    {
+      return FlatArray<T> (data[i].size, data[i].col);
+    }
   };
 
 
-
-
   /// Print table
   template <class T>
   inline ostream & operator<< (ostream & s, const DynamicTable<T> & table)
   {
-    for (int i = 0; i < table.Size(); i++)
+    for (auto i : Range(table))
       {
         s << i << ":";
         for (int j = 0; j < table[i].Size(); j++)
@@ -495,6 +739,69 @@ public:
     return s;
   }
 
+
+  //   Helper function to calculate coloring of a set of indices for parallel processing of independent elements/points/etc.
+  //   Assigns a color to each of colors.Size() elements, such that two elements with the same color don't share a common 'dof',
+  //   the mapping from element to dofs is provided by the function getDofs(int) -> iterable<int>
+  //
+  //   Returns the number of used colors
+  template <typename Tmask>
+  int ComputeColoring( FlatArray<int> colors, size_t ndofs, Tmask const & getDofs)
+  {
+    static Timer timer("ComputeColoring - "+Demangle(typeid(Tmask).name())); RegionTimer rt(timer);
+    static_assert(sizeof(unsigned int)==4, "Adapt type of mask array");
+    size_t n = colors.Size();
+
+    Array<unsigned int> mask(ndofs);
+
+    size_t colored_blocks = 0;
+
+    // We are coloring with 32 colors at once and use each bit to mask conflicts
+    unsigned int check = 0;
+    unsigned int checkbit = 0;
+
+    int current_color = 0;
+    colors = -1;
+    int maxcolor = 0;
+
+    while(colored_blocks<n)
+    {
+        mask = 0;
+        for (auto i : Range(n) )
+        {
+            if(colors[i]>-1) continue;
+            check = 0;
+            const auto & dofs = getDofs(i);
+
+            // Check if adjacent dofs are already marked by current color
+            for (auto dof : dofs)
+                check|=mask[dof];
+
+            // Did we find a free color?
+            if(check != 0xFFFFFFFF)
+            {
+                checkbit = 1;
+                int color = current_color;
+                // find the actual color, which is free (out of 32)
+                while (check & checkbit)
+                {
+                    color++;
+                    checkbit *= 2;
+                }
+                colors[i] = color;
+                maxcolor = color > maxcolor ? color : maxcolor;
+                colored_blocks++;
+                // mask all adjacent dofs with the found color
+                for (auto dof : dofs)
+                    mask[dof] |= checkbit;
+            }
+        }
+        current_color+=32;
+    }
+    return maxcolor+1;
+  }
+
+
   typedef DynamicTable<int> IntTable;
 
 } // namespace ngcore

libsrc/core/taskmanager.cpp

@@ -35,11 +35,7 @@ namespace ngcore
   int TaskManager :: num_threads = 1;
 
   
-  // #ifndef __clang__      
   thread_local int TaskManager :: thread_id = 0;
-  // #else
-  // __thread int TaskManager :: thread_id;
-  // #endif
   
   const function<void(TaskInfo&)> * TaskManager::func;
   const function<void()> * TaskManager::startup_function = nullptr;
@@ -80,14 +76,14 @@ namespace ngcore
     numa_run_on_node (0);
 #endif
 
-#ifndef WIN32
+#if !defined(WIN32) && !defined(EMSCRIPTEN)
     // master has maximal priority !
     int policy;
     struct sched_param param;
     pthread_getschedparam(pthread_self(), &policy, &param);
     param.sched_priority = sched_get_priority_max(policy);
     pthread_setschedparam(pthread_self(), policy, &param);
-#endif // WIN32
+#endif // !defined(WIN32) && !defined(EMSCRIPTEN)
 
     
     task_manager->StartWorkers();
@@ -159,36 +155,33 @@ namespace ngcore
       active_workers = 0;
 
       static int cnt = 0;
-      char buf[100];
       if (use_paje_trace)
-        {
-#ifdef PARALLEL
-          int is_init = -1;
-          MPI_Initialized(&is_init);
-          if (is_init)
-            sprintf(buf, "ng%d_rank%d.trace", cnt++, NgMPI_Comm(MPI_COMM_WORLD).Rank());
-          else
-#endif
-            sprintf(buf, "ng%d.trace", cnt++);
-        }
-      else
-        buf[0] = 0;
-      //sprintf(buf, "");
-      trace = new PajeTrace(num_threads, buf);
+          trace = new PajeTrace(num_threads, "ng" + ToString(cnt++));
     }
 
 
   TaskManager :: ~TaskManager ()
   {
-    delete trace;
-    trace = nullptr;
+    if (use_paje_trace)
+      {
+        delete trace;
+        trace = nullptr;
+      }
     num_threads = 1;
+#ifdef USE_NUMA
+      for (int j = 0; j < num_nodes; j++)
+          numa_free (nodedata[j], sizeof(NodeData));
+#else
+      delete nodedata[0];
+#endif
   }
 
+#ifdef WIN32
   int TaskManager :: GetThreadId()
   {
     return thread_id;
   }
+#endif
   
   void TaskManager :: StartWorkers()
   {
@@ -212,14 +205,14 @@ namespace ngcore
       ;
   }
 
-  static size_t calibrate_init_tsc = __rdtsc();
+  static size_t calibrate_init_tsc = GetTimeCounter();
   typedef std::chrono::system_clock TClock;
   static TClock::time_point calibrate_init_clock = TClock::now();
   
   void TaskManager :: StopWorkers()
   {
     done = true;
-    double delta_tsc = __rdtsc()-calibrate_init_tsc;
+    double delta_tsc = GetTimeCounter()-calibrate_init_tsc;
     double delta_sec = std::chrono::duration<double>(TClock::now()-calibrate_init_clock).count();
     double frequ = (delta_sec != 0) ? delta_tsc/delta_sec : 2.7e9;
     
@@ -248,13 +241,14 @@ namespace ngcore
     const function<void(TaskInfo&)> * func;
     int mynr;
     int total;
+    int producing_thread;
     atomic<int> * endcnt;
 
     TNestedTask () { ; }
     TNestedTask (const function<void(TaskInfo&)> & _func,
                  int _mynr, int _total,
-                 atomic<int> & _endcnt)
-      : func(&_func), mynr(_mynr), total(_total), endcnt(&_endcnt)
+                 atomic<int> & _endcnt, int prod_tid)
+      : func(&_func), mynr(_mynr), total(_total), producing_thread(prod_tid), endcnt(&_endcnt)
     {
       ;
     }
@@ -273,12 +267,14 @@ namespace ngcore
     TPToken ptoken(taskqueue); 
 
     int num = endcnt;
+    auto tid = TaskManager::GetThreadId();
     for (int i = 0; i < num; i++)
-      taskqueue.enqueue (ptoken, { afunc, i, num, endcnt });
+      taskqueue.enqueue (ptoken, { afunc, i, num, endcnt, tid });
   }
 
   bool TaskManager :: ProcessTask()
   {
+    // static Timer t("process task");
     TNestedTask task;
     TCToken ctoken(taskqueue); 
     
@@ -295,8 +291,14 @@ namespace ngcore
           cout << "process nested, nr = " << ti.task_nr << "/" << ti.ntasks << endl;
         }
         */
+        // if(trace && task.producing_thread != ti.thread_nr)
+        // trace->StartTask (ti.thread_nr, t, PajeTrace::Task::ID_TIMER, task.producing_thread);
+
         (*task.func)(ti);
         --*task.endcnt;
+
+        // if(trace && task.producing_thread != ti.thread_nr)
+        // trace->StopTask (ti.thread_nr, t);
         return true;
       }
     return false;
@@ -348,8 +350,28 @@ namespace ngcore
         return;
       }
     
+    if (antasks == 1)
+      {
+        if (trace)
+          trace->StartJob(jobnr, afunc.target_type());
+        jobnr++;
+        if (startup_function) (*startup_function)();
+        TaskInfo ti;
+        ti.task_nr = 0;
+        ti.ntasks = 1;
+        ti.thread_nr = 0; ti.nthreads = 1;
+        {
+          RegionTracer t(ti.thread_nr, jobnr, RegionTracer::ID_JOB, ti.task_nr);
+          afunc(ti);
+        }
+        if (cleanup_function) (*cleanup_function)();
+        if (trace)
+          trace->StopJob();
+        return;
+      }
     
-    trace->StartJob(jobnr, afunc.target_type());
+    if (trace)
+        trace->StartJob(jobnr, afunc.target_type());
 
     func = &afunc;
 
@@ -396,7 +418,7 @@ namespace ngcore
           }
 
       }
-    catch (Exception e)
+    catch (Exception & e)
       {
         {
           lock_guard<mutex> guard(copyex_mutex);
@@ -412,14 +434,19 @@ namespace ngcore
       if (workers_on_node[j])
         {
           while (complete[j] != jobnr)
+          {
+#ifdef NETGEN_ARCH_AMD64
             _mm_pause();
+#endif // NETGEN_ARCH_AMD64
+          }
         }
 
     func = nullptr;
     if (ex)
       throw Exception (*ex);
 
-    trace->StopJob();
+    if (trace)
+        trace->StopJob();
   }
     
   void TaskManager :: Loop(int thd)
@@ -464,6 +491,7 @@ namespace ngcore
 #endif
 
     
+    size_t no_job_counter = 0;
     while (!done)
       {
         if (complete[mynode] > jobdone)
@@ -471,11 +499,12 @@ namespace ngcore
 
         if (jobnr == jobdone)
           {
+            no_job_counter++;
             // RegionTracer t(ti.thread_nr, tCASyield, ti.task_nr);
-            while (ProcessTask()); // do the nested tasks
+            while (ProcessTask()) no_job_counter = 0; // do the nested tasks
                    
-            if(sleep)
-              std::this_thread::sleep_for(std::chrono::microseconds(sleep_usecs));
+            if(sleep || no_job_counter > 10000)
+              std::this_thread::sleep_for(std::chrono::microseconds(10));
             else
               {
 #ifdef WIN32
@@ -516,6 +545,7 @@ namespace ngcore
                 
                 ti.task_nr = mytasks.First()+mytask;
                 ti.ntasks = ntasks;
+                no_job_counter = 0;
                 
                 {
                   RegionTracer t(ti.thread_nr, jobnr, RegionTracer::ID_JOB, ti.task_nr);
@@ -524,7 +554,7 @@ namespace ngcore
               }
 
           }
-        catch (Exception e)
+        catch (Exception & e)
           {
             {
               // cout << "got exception in TM" << endl; 

libsrc/core/taskmanager.hpp

@@ -10,12 +10,13 @@
 #include <atomic>
 #include <functional>
 #include <list>
+#include <cmath>
 #include <ostream>
 #include <thread>
 
 #include "array.hpp"
 #include "paje_trace.hpp"
-#include "profiler.hpp"
+#include "taskmanager.hpp"
 
 #ifdef USE_NUMA
 #include <numa.h>
@@ -78,12 +79,11 @@ namespace ngcore
 
 
 
-    // #ifndef __clang__      
+#ifdef WIN32 // no exported thread_local in dlls on Windows
     static thread_local int thread_id;
-    // #else
-    // static __thread int thread_id;
-    // #endif
-    
+#else
+    NGCORE_API static thread_local int thread_id;
+#endif
     NGCORE_API static bool use_paje_trace;
   public:
     
@@ -102,11 +102,15 @@ namespace ngcore
     void ResumeWorkers() { sleep = false; }
 
     NGCORE_API static void SetNumThreads(int amax_threads);
-    NGCORE_API static int GetMaxThreads() { return max_threads; }
+    static int GetMaxThreads() { return max_threads; }
     // static int GetNumThreads() { return task_manager ? task_manager->num_threads : 1; }
-    NGCORE_API static int GetNumThreads() { return num_threads; }
+    static int GetNumThreads() { return num_threads; }
+#ifdef WIN32
     NGCORE_API static int GetThreadId();
-    NGCORE_API int GetNumNodes() const { return num_nodes; }
+#else
+    static int GetThreadId() { return thread_id; }
+#endif
+    int GetNumNodes() const { return num_nodes; }
 
     static void SetPajeTrace (bool use)  { use_paje_trace = use; }
     
@@ -313,6 +317,7 @@ namespace ngcore
     
   public:
     SharedLoop (IntRange ar) : r(ar) { cnt = r.begin(); }
+    SharedLoop (size_t s) : SharedLoop (IntRange{s}) { ; }
     SharedIterator begin() { return SharedIterator (cnt, r.end(), true); }
     SharedIterator end()   { return SharedIterator (cnt, r.end(), false); }
   };
@@ -459,25 +464,32 @@ public:
     //   return IntRange(begin, end);
     // }
     
-    bool PopFirst (size_t & first)
+    bool PopFirst (size_t & hfirst)
     {
-      first = begin++;
-      return first < end;
-      /*
-      // int oldbegin = begin;
-      size_t oldbegin = begin.load(std::memory_order_acquire);
-      if (oldbegin >= end) return false;
-      while (!begin.compare_exchange_weak (oldbegin, oldbegin+1,
-                                           std::memory_order_relaxed, std::memory_order_relaxed))
-        if (oldbegin >= end) return false;
+      // first = begin++;
+      // return first < end;
       
-      first = oldbegin;
-      return true;
-      */
+      size_t first = begin.load(std::memory_order_relaxed);
+      
+      size_t nextfirst = first+1;
+      if (first >= end) nextfirst = std::numeric_limits<size_t>::max()-1;
+
+      // while (!begin.compare_exchange_weak (first, nextfirst))
+      while (!begin.compare_exchange_weak (first, nextfirst,
+                                           std::memory_order_relaxed,
+                                           std::memory_order_relaxed))
+        {
+          first = begin;
+          nextfirst = first+1;
+          if (nextfirst >= end) nextfirst = std::numeric_limits<size_t>::max()-1;
+        }
+      hfirst = first;
+      return first < end;
     }
     
     bool PopHalf (IntRange & r)
     {
+      /*
       // int oldbegin = begin;
       size_t oldbegin = begin.load(std::memory_order_acquire);
       size_t oldend = end.load(std::memory_order_acquire);
@@ -493,6 +505,28 @@ public:
       
       r = IntRange(oldbegin, (oldbegin+oldend+1)/2);
       return true;
+      */
+
+
+      size_t oldbegin = begin; // .load(std::memory_order_acquire);
+      size_t oldend = end; // .load(std::memory_order_acquire);
+      if (oldbegin >= oldend) return false;
+      
+      size_t nextbegin = (oldbegin+oldend+1)/2;
+      if (nextbegin >= oldend) nextbegin = std::numeric_limits<size_t>::max()-1;
+      
+      while (!begin.compare_exchange_weak (oldbegin, nextbegin))
+        // std::memory_order_relaxed, std::memory_order_relaxed))
+        {
+          oldend = end; // .load(std::memory_order_acquire);
+          if (oldbegin >= oldend) return false;
+          
+          nextbegin = (oldbegin+oldend+1)/2;
+          if (nextbegin >= oldend) nextbegin = std::numeric_limits<size_t>::max()-1;
+        }
+      
+      r = IntRange(oldbegin, (oldbegin+oldend+1)/2);
+      return true;
     }
   };
   
@@ -590,6 +624,8 @@ public:
       Reset (r);
     }
 
+    SharedLoop2 (size_t s) : SharedLoop2 (IntRange{s}) { } 
+    
     void Reset (IntRange r)
     {
       for (size_t i = 0; i < ranges.Size(); i++)
@@ -599,6 +635,9 @@ public:
       participants.store(0, std::memory_order_relaxed);
       processed.store(0, std::memory_order_release);
     }
+
+    void Reset (size_t s) { Reset(IntRange{s}); }
+      
     
     SharedIterator begin()
     {
@@ -853,6 +892,30 @@ public:
   auto GetNum() const { return num; } 
 };
 
+
+/*
+  // some idea, not yet supported
+
+ using namespace std;
+  template <typename T>
+  class ParallelValue
+  {
+    T val;
+  public:
+    ParallelValue (const T & _val) : val(_val) { ; }
+    operator T () const { return val; }
+  };
+  
+  template <typename FUNC> class ParallelFunction
+  {
+    FUNC f;
+  public:
+    ParallelFunction (const FUNC & _f) : f(_f) { ; }
+    operator FUNC () const { return f; }
+    auto operator() (size_t i) const { return f(i); }
+  };
+*/
+
 /* currently not used, plus causing problems on MSVC 2017
 template <typename T, typename std::enable_if<ngstd::has_call_operator<T>::value, int>::type = 0>                                  
 inline ParallelFunction<T> operator| (const T & func, Tasks tasks)
@@ -954,7 +1017,7 @@ public:
     int num_nodes = numa_num_configured_nodes();
     size_t pagesize = numa_pagesize();
     
-    int npages = ceil ( double(s)*sizeof(T) / pagesize );
+    int npages = std::ceil ( double(s)*sizeof(T) / pagesize );
 
     // cout << "size = " << numa_size << endl;
     // cout << "npages = " << npages << endl;
@@ -1058,68 +1121,6 @@ public:
   
 #endif // USE_NUMA
 
-  //   Helper function to calculate coloring of a set of indices for parallel processing of independent elements/points/etc.
-  //   Assigns a color to each of colors.Size() elements, such that two elements with the same color don't share a common 'dof',
-  //   the mapping from element to dofs is provided by the function getDofs(int) -> iterable<int>
-  //
-  //   Returns the number of used colors
-  template <typename Tmask>
-  int ComputeColoring( FlatArray<int> colors, size_t ndofs, Tmask const & getDofs)
-  {
-    static Timer timer("ComputeColoring - "+Demangle(typeid(Tmask).name())); RegionTimer rt(timer);
-    static_assert(sizeof(unsigned int)==4, "Adapt type of mask array");
-    auto n = colors.Size();
-
-    Array<unsigned int> mask(ndofs);
-
-    int colored_blocks = 0;
-
-    // We are coloring with 32 colors at once and use each bit to mask conflicts
-    unsigned int check = 0;
-    unsigned int checkbit = 0;
-
-    int current_color = 0;
-    colors = -1;
-    int maxcolor = 0;
-
-    while(colored_blocks<n)
-    {
-        mask = 0;
-        for (auto i : Range(n) )
-        {
-            if(colors[i]>-1) continue;
-            check = 0;
-            const auto & dofs = getDofs(i);
-
-            // Check if adjacent dofs are already marked by current color
-            for (auto dof : dofs)
-                check|=mask[dof];
-
-            // Did we find a free color?
-            if(check != 0xFFFFFFFF)
-            {
-                checkbit = 1;
-                int color = current_color;
-                // find the actual color, which is free (out of 32)
-                while (check & checkbit)
-                {
-                    color++;
-                    checkbit *= 2;
-                }
-                colors[i] = color;
-                maxcolor = color > maxcolor ? color : maxcolor;
-                colored_blocks++;
-                // mask all adjacent dofs with the found color
-                for (auto dof : dofs)
-                    mask[dof] |= checkbit;
-            }
-        }
-        current_color+=32;
-    }
-    return maxcolor+1;
-  }
-
-
 }
 
 

libsrc/core/type_traits.hpp

@@ -16,6 +16,15 @@ namespace ngcore
     template<typename T>
     struct is_any_pointer_impl : std::false_type {};
 
+    // check if second template argument is tuple of base classes to first
+    // template argument, return constexpr bool
+    template<typename T, typename Tuple>
+    constexpr bool is_base_of_tuple = false;
+
+    template<typename T, typename... Ts>
+    constexpr bool is_base_of_tuple<T, std::tuple<Ts...>> =
+      all_of_tmpl<std::is_base_of<Ts, T>::value...>;
+
     template<typename T>
     struct is_any_pointer_impl<T*> : std::true_type {};
 

libsrc/core/utils.cpp

@@ -1,19 +1,79 @@
+#include "ngcore_api.hpp"
 #include "utils.hpp"
 #include "logging.hpp"
+#include "simd_generic.hpp"
 
-#ifndef WIN32
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#undef WIN32_LEAN_AND_MEAN
+#else // WIN32
 #include <cxxabi.h>
-#endif
+#include <dlfcn.h>
+#endif //WIN32
+//
+#include <array>
+#include <filesystem>
 #include <iostream>
+#include <regex>
+#include <string>
+#include <thread>
+
+#include "ngstream.hpp"
+
 
 namespace ngcore
 {
+    namespace detail
+    {
+        // see https://github.com/RobotLocomotion/drake/blob/master/common/nice_type_name.cc
+        static const auto demangle_regexes =
+            std::array<std::pair<std::regex, std::string>, 8>{
+                // Remove unwanted keywords and following space. (\b is word boundary.)
+                std::make_pair(std::regex("\\b(class|struct|enum|union) "), ""),
+                // Tidy up anonymous namespace.
+                {std::regex("[`(]anonymous namespace[')]"), "(anonymous)"},
+                // Replace Microsoft __int64 with long long.
+                {std::regex("\\b__int64\\b"), "long long"},
+                // Temporarily replace spaces we want to keep with "!". (\w is
+                // alphanumeric or underscore.)
+                {std::regex("(\\w) (\\w)"), "$1!$2"},
+                {std::regex(" "), ""},  // Delete unwanted spaces.
+                // Some compilers throw in extra namespaces like "__1" or "__cxx11".
+                // Delete them.
+                {std::regex("\\b__[[:alnum:]_]+::"), ""},
+                {std::regex("!"), " "},  // Restore wanted spaces.
+
+                // Recognize std::string's full name and abbreviate.
+                {std::regex("\\bstd::basic_string<char,std::char_traits<char>,"
+                        "std::allocator<char>>"), "std::string"}
+            };
+        std::string CleanupDemangledName( std::string s )
+        {
+            for(const auto & [r, sub] : demangle_regexes)
+                s = std::regex_replace (s,r,sub);
+#ifdef EMSCRIPTEN
+            // for some reason regex_replace is not working at all
+            std::string temp = s;
+            s = "";
+            for(auto c : temp)
+              if(c!=' ')
+                s+=c;
+#endif // EMSCRIPTEN
+
+            return s;
+        }
+    } // namespace detail
+
   // parallel netgen
   int id = 0, ntasks = 1;
 
 #ifdef WIN32
   // windows does demangling in typeid(T).name()
-  NGCORE_API std::string Demangle(const char* typeinfo) { return typeinfo; }
+  NGCORE_API std::string Demangle(const char* typeinfo) {
+      std::string name = typeinfo;
+      return detail::CleanupDemangledName(name);
+  }
 #else
   NGCORE_API std::string Demangle(const char* typeinfo)
   {
@@ -21,15 +81,23 @@ namespace ngcore
     try
       {
         char *s = abi::__cxa_demangle(typeinfo, nullptr, nullptr, &status);
-        std::string result{s};
-        free(s);
+        std::string result;
+        if (s == nullptr)
+          result = typeinfo;
+        else
+          {
+            result = s;
+            free(s);
+          }
+        result = detail::CleanupDemangledName(result);
         return result;
       }
     catch( const std::exception & e )
       {
         GetLogger("utils")->warn("{}:{} cannot demangle {}, status: {}, error:{}", __FILE__, __LINE__, typeinfo, status, e.what());
       }
-    return typeinfo;
+    std::string name = typeinfo;
+    return detail::CleanupDemangledName(name);
   }
 #endif
 
@@ -50,5 +118,116 @@ namespace ngcore
 
   const std::chrono::time_point<TClock> wall_time_start = TClock::now();
 
+  int printmessage_importance = getenv("NG_MESSAGE_LEVEL") ? atoi(getenv("NG_MESSAGE_LEVEL")) : 0;
+  bool NGSOStream :: glob_active = true;
+
+  NGCORE_API int GetCompiledSIMDSize()
+  {
+      return GetDefaultSIMDSize();
+  }
+
+  NGCORE_API bool IsRangeCheckEnabled()
+  {
+#ifdef NETGEN_ENABLE_CHECK_RANGE
+      return true;
+#else
+      return false;
+#endif
+  }
+
+  NGCORE_API std::filesystem::path GetTempFilename()
+  {
+      static int counter = 0;
+      auto path = std::filesystem::temp_directory_path();
+      path += ".temp_netgen_file_"+ToString(counter++)+"_"+ToString(GetTimeCounter());
+      return path;
+  }
+
+
+  SharedLibrary :: SharedLibrary(const std::filesystem::path & lib_name_, std::optional<std::filesystem::path> directory_to_delete_, bool global )
+      : lib_name(lib_name_),directory_to_delete(directory_to_delete_)
+  {
+    Load(lib_name, global);
+  }
+
+  SharedLibrary :: ~SharedLibrary()
+  {
+    Unload();
+    if(directory_to_delete)
+      for([[maybe_unused]] auto i : Range(5))
+      {
+        // on Windows, a (detached?) child process of the compiler/linker might still block the directory
+        // wait for it to finish (up to a second)
+        try
+        {
+          std::filesystem::remove_all(*directory_to_delete);
+          directory_to_delete = std::nullopt;
+          break;
+        }
+        catch(const std::exception &e)
+        {
+          std::this_thread::sleep_for(std::chrono::milliseconds(200));
+        }
+      }
+    if(directory_to_delete)
+      std::cerr << "Could not delete " << directory_to_delete->string() << std::endl;
+  }
+
+  void SharedLibrary :: Load( const std::filesystem::path & lib_name_, bool global )
+  {
+    Unload();
+    lib_name = lib_name_;
+#ifdef WIN32
+    lib = LoadLibraryW(lib_name.wstring().c_str());
+    if (!lib) throw std::runtime_error(std::string("Could not load library ") + lib_name.string());
+#else // WIN32
+    auto flags = RTLD_NOW;
+    if (global) flags |= RTLD_GLOBAL;
+    lib = dlopen(lib_name.c_str(), flags);
+    if(lib == nullptr) throw std::runtime_error(dlerror());
+#endif // WIN32
+  }
+
+  void SharedLibrary :: Unload() {
+    if(lib)
+    {
+#ifdef WIN32
+      FreeLibrary((HMODULE)lib);
+#else // WIN32
+      int rc = dlclose(lib);
+      if(rc != 0) std::cerr << "Failed to close library " << lib_name << std::endl;
+#endif // WIN32
+    }
+  }
+
+  void* SharedLibrary :: GetRawSymbol( std::string func_name )
+  {
+#ifdef WIN32
+    void* func = GetProcAddress((HMODULE)lib, func_name.c_str());
+    if(func == nullptr)
+      throw std::runtime_error(std::string("Could not find function ") + func_name + " in library " + lib_name.string());
+#else // WIN32
+    void* func = dlsym(lib, func_name.c_str());
+    if(func == nullptr)
+        throw std::runtime_error(dlerror());
+#endif // WIN32
+
+    return func;
+  }
+
+  void* GetRawSymbol( std::string func_name )
+  {
+    void * func = nullptr;
+#ifdef WIN32
+    throw std::runtime_error("GetRawSymbol not implemented on WIN32");
+#else // WIN32
+    func = dlsym(RTLD_DEFAULT, func_name.c_str());
+    if(func == nullptr)
+        throw std::runtime_error(dlerror());
+#endif // WIN32
+    return func;
+  }
+
+
 } // namespace ngcore
 

libsrc/core/utils.hpp

@@ -3,23 +3,34 @@
 
 #include <atomic>
 #include <chrono>
+#include <filesystem>
 #include <map>
 #include <ostream>
+#include <optional>
 #include <sstream>
 #include <string>
 
+#include "ngcore_api.hpp"       // for NGCORE_API and CPU arch macros
+
+#if defined(__APPLE__) && defined(NETGEN_ARCH_ARM64)
+#include <mach/mach_time.h>
+#endif
+
+#ifdef NETGEN_ARCH_AMD64
 #ifdef WIN32
 #include <intrin.h>   // for __rdtsc()  CPU time step counter
 #else
 #include <x86intrin.h>   // for __rdtsc()  CPU time step counter
 #endif // WIN32
-
-#include "ngcore_api.hpp"       // for NGCORE_API
+#endif // NETGEN_ARCH_AMD64
 
 namespace ngcore
 {
   // MPI rank, nranks TODO: Rename
-  extern NGCORE_API int id, ntasks;
+  // [[deprecated("don't use global id/ntasks")]]       
+  extern NGCORE_API int id;
+  // [[deprecated("don't use global id/ntasks")]]         
+  extern NGCORE_API int ntasks;
   
   NGCORE_API std::string Demangle(const char* typeinfo);
 
@@ -52,7 +63,21 @@ namespace ngcore
 
   inline TTimePoint GetTimeCounter() noexcept
   {
-      return TTimePoint(__rdtsc());
+#if defined(__APPLE__) && defined(NETGEN_ARCH_ARM64)
+    return mach_absolute_time();
+#elif defined(NETGEN_ARCH_AMD64)
+    return __rdtsc();
+#elif defined(NETGEN_ARCH_ARM64) && defined(__GNUC__)
+    // __GNUC__ is also defined by CLANG. Use inline asm to read Generic Timer
+    unsigned long long tics;
+    __asm __volatile("mrs %0, CNTVCT_EL0" : "=&r" (tics));
+    return tics;
+#elif defined(__EMSCRIPTEN__)
+    return std::chrono::high_resolution_clock::now().time_since_epoch().count();
+#else
+#warning "Unsupported CPU architecture"
+    return 0;
+#endif
   }
 
   template <class T>
@@ -63,6 +88,42 @@ namespace ngcore
       return ss.str();
   }
 
+  inline std::string ToLower( const std::string & s )
+  {
+    std::string res;
+    res.reserve(s.size());
+
+    for(auto & c : s)
+        res.push_back(tolower(c));
+
+    return res;
+  }
+
+  inline std::string ToLower( const std::filesystem::path & p )
+  {
+    return ToLower(p.string());
+  }
+
+
+
+  template <class T>
+  void SaveBin (std::ostream & ost, const T & val)
+  {
+    const char * cp = reinterpret_cast<const char*> (&val);
+    for (unsigned j = 0; j < sizeof(T); j++)
+      ost.put(cp[j]);
+  }
+  
+  
+  template <class T>
+  void LoadBin (std::istream & ist, T & val)
+  {
+    char * cp = reinterpret_cast<char*> (&val);
+    for (unsigned j = 0; j < sizeof(T); j++)
+      ist.get(cp[j]);
+  }
+  
+  
   template<typename T1, typename T2>
   std::ostream& operator << (std::ostream& ost, const std::map<T1,T2>& map)
   {
@@ -79,6 +140,65 @@ namespace ngcore
       b = std::move(temp);
   }
 
+  
+  /// min of 2 values
+  template <class T>
+  NETGEN_INLINE T min2 (T a, T b)
+  {
+    return (a < b) ? a : b;
+  }
+  
+  /// max of 2 values
+  template <class T>
+  NETGEN_INLINE T max2 (T a, T b)
+  {
+    return (a > b) ? a : b;
+}
+  
+  /// min of 3 values
+  template <class T>
+  NETGEN_INLINE T min3 (T a, T b, T c)
+  {
+  return (a < b) ? (a < c) ? a : c
+    : (b < c) ? b : c;
+  }
+  
+  /// max of 3 values
+  template <class T>
+  NETGEN_INLINE T max3 (T a, T b, T c)
+  {
+    ///
+    return (a > b) ? ((a > c) ? a : c)
+      : ((b > c) ? b : c);
+  }
+  
+  
+  /// sign of value (+1, 0, -1)
+  template <class T>
+  NETGEN_INLINE int sgn (T a)
+  {
+    return (a > 0) ? 1 : ( ( a < 0) ? -1 : 0 );
+  }
+  
+  /// square element 
+  template <class T>
+  NETGEN_INLINE constexpr T sqr (const T a)
+  {
+    return a * a; 
+  }
+  
+  /// element to the third power
+  template <class T>
+  NETGEN_INLINE T pow3 (const T a)
+  {
+    return a * a * a; 
+  }
+  
+  
+
+  NETGEN_INLINE double IfPos (double a, double b, double c) { return a>0 ? b : c; }
+  NETGEN_INLINE double IfZero (double a, double b, double c) { return a==0. ? b : c; }
+
   // checks if string starts with sequence
   inline bool StartsWith(const std::string& str, const std::string& start)
   {
@@ -130,6 +250,49 @@ namespace ngcore
       return current;
   }
 
+  
+
+  
+  template <int N> using IC = std::integral_constant<int,N>;  // needed for Iterate
+
+  
+  namespace detail {
+    template <typename T, typename Enable = int>
+    struct IsIC_trait {
+      static constexpr auto check() { return false; }
+    };
+    
+    template <typename T>
+    struct IsIC_trait<T, std::enable_if_t<std::is_same_v<T, IC<T::value>> == true, int> > {
+      static constexpr auto check() { return true; }  
+    };
+  }
+  
+  template <typename T>
+  constexpr bool is_IC() {
+    return detail::IsIC_trait<T>::check();
+  }
+  
+  
+
+  
+  template <int NUM, typename FUNC>
+  NETGEN_INLINE void Iterate (FUNC f)
+  {
+    if constexpr (NUM > 1) Iterate<NUM-1> (f);
+  if constexpr (NUM >= 1) f(IC<NUM-1>());
+  }
+  
+  
+  template <int NUM, typename FUNC>
+  NETGEN_INLINE void Switch (size_t nr, FUNC f)
+  {
+    if (NUM-1 == nr) f(IC<NUM-1>());
+    if constexpr (NUM > 1) Switch<NUM-1> (nr, f);
+  }
+  
+  
+  
   namespace detail
   {
     template<typename T>
@@ -161,7 +324,9 @@ namespace ngcore
       while (!m.compare_exchange_weak(should, true))
         {
           should = false;
+#ifdef NETGEN_ARCH_AMD64
           _mm_pause();
+#endif // NETGEN_ARCH_AMD64
         }
     }
     void unlock()
@@ -178,6 +343,45 @@ namespace ngcore
     ~MyLock () { mutex.unlock(); }
   };
 
+  NGCORE_API int GetCompiledSIMDSize();
+  NGCORE_API bool IsRangeCheckEnabled();
+
+  NGCORE_API std::filesystem::path GetTempFilename();
+
+  NGCORE_API void* GetRawSymbol( std::string func_name );
+
+  template <typename TFunc>
+  TFunc GetSymbol( std::string func_name )
+  {
+    return reinterpret_cast<TFunc>(GetRawSymbol(func_name));
+  }
+
+  // Class to handle/load shared libraries
+  class NGCORE_API SharedLibrary
+  {
+    std::filesystem::path lib_name;
+    std::optional<std::filesystem::path> directory_to_delete = std::nullopt;
+    void *lib = nullptr;
+
+  public:
+    SharedLibrary() = default;
+    SharedLibrary(const std::filesystem::path & lib_name_, std::optional<std::filesystem::path> directory_to_delete_ = std::nullopt, bool global = false );
+
+    SharedLibrary(const SharedLibrary &) = delete;
+    SharedLibrary & operator =(const SharedLibrary &) = delete;
+
+    ~SharedLibrary();
+
+    template <typename TFunc>
+    TFunc GetSymbol( std::string func_name )
+    {
+      return reinterpret_cast<TFunc>(GetRawSymbol(func_name));
+    }
+
+    void Load( const std::filesystem::path & lib_name_, bool global = true);
+    void Unload();
+    void* GetRawSymbol( std::string func_name );
+  };
 
 } // namespace ngcore
 

libsrc/core/version.cpp

@@ -0,0 +1,29 @@
+#include <map>
+
+#include <netgen_version.hpp>
+#include "exception.hpp"
+#include "version.hpp"
+
+namespace ngcore
+{
+  // clang-tidy should ignore this static object
+  static std::map<std::string, VersionInfo> library_versions;  // NOLINT
+
+  const VersionInfo& GetLibraryVersion(const std::string& library)
+  { return library_versions[library]; }
+
+  const std::map<std::string, VersionInfo>& GetLibraryVersions()
+  { return library_versions; }
+
+  void SetLibraryVersion(const std::string& library, const VersionInfo& version)
+  {
+    if(library_versions.count(library) && (library_versions[library] != version))
+      throw Exception("Failed to set library version for " + library + " to " + version.to_string() + ": version already set to " + library_versions[library].to_string());
+    library_versions[library] = version;
+  }
+
+  static bool dummy = [](){
+    SetLibraryVersion("netgen", NETGEN_VERSION);
+    return true;
+  }();
+} // namespace ngcore

libsrc/core/version.hpp

@@ -80,6 +80,10 @@ namespace ngcore
       return mayor_ == other.mayor_ && minor_ == other.minor_ && release == other.release
         && patch == other.patch;
     }
+    bool operator !=(const VersionInfo& other) const
+    {
+      return !(*this==other);
+    }
     bool operator >(const VersionInfo& other) const { return other < (*this); }
     bool operator <=(const VersionInfo& other) const { return !((*this) > other); }
     bool operator >=(const VersionInfo& other) const { return !((*this) < other); }
@@ -89,6 +93,10 @@ namespace ngcore
   {
     return ost << version.to_string();
   }
+
+  NGCORE_API const VersionInfo& GetLibraryVersion(const std::string& library);
+  NGCORE_API const std::map<std::string, VersionInfo>& GetLibraryVersions();
+  NGCORE_API void SetLibraryVersion(const std::string& library, const VersionInfo& version);
 } // namespace ngcore
 
 #endif // NETGEN_CORE_VERSION_HPP

libsrc/core/xbool.hpp

@@ -20,16 +20,18 @@ namespace ngcore
 
   public:
     xbool (bool b) : state(b ? 2 : 0) { ; }
-    xbool (TMAYBE x) : state(1) { ; }
+    xbool (TMAYBE /* x */) : state(1) { ; }
     xbool () = default;
     xbool (const xbool &) = default;
     
     xbool & operator= (bool b) { state = b ? 2 : 0; return *this; }
-    xbool & operator= (TMAYBE x) { state = 1; return *this; }
+    xbool & operator= (TMAYBE /* x */) { state = 1; return *this; }
     
     bool IsTrue () const { return state == 2; }
     bool IsMaybe () const { return state == 1; }
     bool IsFalse () const { return state == 0; }
+    bool IsMaybeTrue() const { return state >= 1; } 
+    bool IsMaybeFalse() const { return state <= 1; } 
     friend ostream & operator<< (ostream & ost, xbool xb);
   };
 

libsrc/csg/CMakeLists.txt

@@ -1,33 +1,14 @@
-add_definitions(-DNGINTERFACE_EXPORTS)
-add_library(csg ${NG_LIB_TYPE}
+target_sources(nglib PRIVATE
         algprim.cpp brick.cpp   
         bspline2d.cpp csgeom.cpp csgparser.cpp curve2d.cpp edgeflw.cpp	       
         explicitcurve2d.cpp extrusion.cpp gencyl.cpp genmesh.cpp identify.cpp  
         manifold.cpp meshsurf.cpp polyhedra.cpp revolution.cpp singularref.cpp 
         solid.cpp specpoin.cpp spline3d.cpp surface.cpp triapprox.cpp zrefine.cpp 
         python_csg.cpp splinesurface.cpp
-        )
-if(APPLE)
-    set_target_properties( csg PROPERTIES SUFFIX ".so")
-endif(APPLE)
-
-target_link_libraries(csg PUBLIC mesh PRIVATE netgen_python)
-if(NOT WIN32)
-  install( TARGETS csg ${NG_INSTALL_DIR})
-endif(NOT WIN32)
-
-target_link_libraries(csg PUBLIC ngcore)
+)
 
 if(USE_GUI)
-    add_library(csgvis ${NG_LIB_TYPE} vscsg.cpp )
-    target_link_libraries(csgvis PRIVATE netgen_python PUBLIC ngcore)
-    if(NOT WIN32)
-        target_link_libraries(csgvis PUBLIC csg visual)
-        if(APPLE)
-            set_target_properties( csgvis PROPERTIES SUFFIX ".so")
-        endif(APPLE)
-        install( TARGETS csgvis ${NG_INSTALL_DIR})
-    endif(NOT WIN32)
+    target_sources(nggui PRIVATE vscsg.cpp csgpkg.cpp)
 endif(USE_GUI)
 
 install(FILES

libsrc/csg/algprim.cpp

@@ -1,4 +1,5 @@
 #include <mystdlib.h>
+#include <core/register_archive.hpp>
 
 
 #include <linalg.hpp>
@@ -877,9 +878,11 @@ namespace netgen
     Vec<3> v1 = b - a;
     Vec<3> v2 = cyl2->a - a;
 
-    if ( fabs (v1 * v2) < (1-eps) * v1.Length() * v2.Length()) return 0;
+    // if ( fabs (v1 * v2) < (1-1e-12) * v1.Length() * v2.Length()) return 0;
+    if ( Cross(v1,v2).Length2() > 1e-20 * v1.Length2() * v2.Length2()) return 0;
     v2 = cyl2->b - a;
-    if ( fabs (v1 * v2) < (1-eps) * v1.Length() * v2.Length()) return 0;
+    // if ( fabs (v1 * v2) < (1-eps) * v1.Length() * v2.Length()) return 0;
+    if ( Cross(v1,v2).Length2() > 1e-20 * v1.Length2() * v2.Length2()) return 0;
 
     inv = 0;
     return 1;

libsrc/csg/brick.cpp

@@ -1,4 +1,5 @@
 #include <mystdlib.h>
+#include <core/register_archive.hpp>
 
 #include <linalg.hpp>
 #include <csg.hpp>

libsrc/csg/csgeom.cpp

@@ -1,5 +1,6 @@
 #include <mystdlib.h>
 #include <myadt.hpp>
+#include <core/register_archive.hpp>
 
 #include <linalg.hpp>
 #include <csg.hpp>
@@ -130,6 +131,7 @@ namespace netgen
                Point<3> & newp, EdgePointGeomInfo & newgi) const
   {
     Point<3> hnewp = p1+secpoint*(p2-p1);
+
     //(*testout) << "hnewp " << hnewp << " s1 " << surfi1 << " s2 " << surfi2 << endl;
     if (surfi1 != -1 && surfi2 != -1 && surfi1 != surfi2)
       {
@@ -175,9 +177,8 @@ namespace netgen
     
     solids.DeleteAll ();
 
-    for (int i = 0; i < splinecurves2d.Size(); i++)
-      delete splinecurves2d[i];
     splinecurves2d.DeleteAll();
+    splinecurves3d.DeleteAll();
     
     /*
     for (int i = 0; i < surfaces.Size(); i++)
@@ -220,7 +221,16 @@ namespace netgen
 
   int CSGeometry :: GenerateMesh (shared_ptr<Mesh> & mesh, MeshingParameters & mparam)
   {
-    return CSGGenerateMesh (*this, mesh, mparam);
+    if(restricted_h.Size())
+      {
+        // copy so that we don't change mparam outside
+        MeshingParameters mp = mparam;
+        for(const auto& [pnt, maxh] : restricted_h)
+          mp.meshsize_points.Append({pnt, maxh});
+        return CSGGenerateMesh (*this, mesh, mp);
+      }
+    else
+      return CSGGenerateMesh (*this, mesh, mparam);
   }
   
   class WritePrimitivesIt : public SolidIterator
@@ -254,7 +264,7 @@ namespace netgen
   }
 
 
-  void CSGeometry :: Save (string filename) const
+  void CSGeometry :: Save (const filesystem::path & filename) const
   {
     ofstream ost (filename.c_str());
     Save (ost);
@@ -312,7 +322,7 @@ namespace netgen
   {
     //  CSGeometry * geo = new CSGeometry;
   
-    char key[100], name[100], classname[100], sname[100];
+    char key[100], name[100], classname[100], sname[150];
     int ncoeff, i, j;
     NgArray<double> coeff;
 
@@ -339,7 +349,7 @@ namespace netgen
 
 	    for (j = 0; j < nprim->GetNSurfaces(); j++)
 	      {
-		sprintf (sname, "%s,%d", name, j);
+		snprintf (sname, size(sname), "%s,%d", name, j);
 		AddSurface (sname, &nprim->GetSurface(j));
 		nprim -> SetSurfaceId (j, GetNSurf());
 	      }
@@ -407,7 +417,7 @@ namespace netgen
       & identpoints & boundingbox & isidenticto & ideps
       & filename & spline_surfaces & splinecurves2d & splinecurves3d & surf2prim;
     if(archive.Input())
-      FindIdenticSurfaces(1e-6);
+      FindIdenticSurfaces(1e-8 * MaxSize());
   }
 
   void CSGeometry :: SaveSurfaces (ostream & out) const
@@ -622,8 +632,8 @@ namespace netgen
   {
     static int cntsurfs = 0;
     cntsurfs++;
-    char name[15];
-    sprintf (name, "nnsurf%d", cntsurfs);
+    char name[20];
+    snprintf (name, size(name), "nnsurf%d", cntsurfs);
     AddSurface (name, surf);
   }
  
@@ -711,24 +721,24 @@ namespace netgen
 
 
 
-  void CSGeometry :: SetSplineCurve (const char * name, SplineGeometry<2> * spl)
+  void CSGeometry :: SetSplineCurve (const char * name, shared_ptr<SplineGeometry<2>> spl)
   {
     splinecurves2d.Set(name,spl);
   }
-  void CSGeometry :: SetSplineCurve (const char * name, SplineGeometry<3> * spl)
+  void CSGeometry :: SetSplineCurve (const char * name, shared_ptr<SplineGeometry<3>> spl)
   {
     splinecurves3d.Set(name,spl);
   }
 
 
-  const SplineGeometry<2> * CSGeometry :: GetSplineCurve2d (const string & name) const
+  shared_ptr<SplineGeometry<2>> CSGeometry :: GetSplineCurve2d (const string & name) const
   {
     if (splinecurves2d.Used(name))
       return splinecurves2d[name];
     else
       return NULL;
   }
-  const SplineGeometry<3> * CSGeometry :: GetSplineCurve3d (const string & name) const
+  shared_ptr<SplineGeometry<3>> CSGeometry :: GetSplineCurve3d (const string & name) const
   {
     if (splinecurves3d.Used(name))
       return splinecurves3d[name];
@@ -939,6 +949,7 @@ namespace netgen
   {
     int inv;
     int nsurf = GetNSurf();
+    identicsurfaces.DeleteData();
 
 
     isidenticto.SetSize(nsurf);
@@ -1200,7 +1211,7 @@ namespace netgen
 	PrintMessage (2, "Object ", i, " has ", tams->GetNT(), " triangles");
       }
       }
-    catch (exception)
+    catch (const std::exception &)
       {
 	cerr << "*************************************************************" << endl
 	     << "****   out of memory problem in CSG visualization        ****" << endl
@@ -1609,21 +1620,21 @@ namespace netgen
   class CSGeometryRegister : public GeometryRegister
   {
   public:
-    virtual NetgenGeometry * Load (string filename) const;
-    virtual NetgenGeometry * LoadFromMeshFile (istream & ist) const;
+    virtual NetgenGeometry * Load (const filesystem::path & filename) const;
+    virtual NetgenGeometry * LoadFromMeshFile (istream & ist, string token) const;
     // virtual VisualScene * GetVisualScene (const NetgenGeometry * geom) const;
   };
 
   extern CSGeometry * ParseCSG (istream & istr, CSGeometry *instance=nullptr);
 
-  NetgenGeometry *  CSGeometryRegister :: Load (string filename) const
+  NetgenGeometry *  CSGeometryRegister :: Load (const filesystem::path & filename) const
   {
-    const char * cfilename = filename.c_str();
-    if (strcmp (&cfilename[strlen(cfilename)-3], "geo") == 0)
+    string extension = filename.extension().string();
+    if (extension == ".geo")
       {
-	PrintMessage (1, "Load CSG geometry file ", cfilename);
+	PrintMessage (1, "Load CSG geometry file ", filename);
 
-	ifstream infile(cfilename);
+	ifstream infile(filename);
 
 	CSGeometry * hgeom = ParseCSG (infile);
 	if (!hgeom)
@@ -1633,38 +1644,28 @@ namespace netgen
 	return hgeom;
       }
 
-    if (strcmp (&cfilename[strlen(cfilename)-3], "ngg") == 0)
+    if (extension == ".ngg")
       {
-	PrintMessage (1, "Load new CSG geometry file ", cfilename);
+	PrintMessage (1, "Load new CSG geometry file ", filename);
 
-	ifstream infile(cfilename);
+	ifstream infile(filename);
 	CSGeometry * hgeom = new CSGeometry("");
 	hgeom -> Load (infile);
 
 	return hgeom;
       }
-
-
     
     return NULL;
   }
 
-  NetgenGeometry * CSGeometryRegister :: LoadFromMeshFile (istream & ist) const 
+  NetgenGeometry * CSGeometryRegister :: LoadFromMeshFile (istream & ist, string token) const
   {
-    string auxstring;
-    if (ist.good())
-      {
-	ist >> auxstring;
-	if (auxstring == "csgsurfaces")
-	  {
-	    CSGeometry * geometry = new CSGeometry ("");
-	    geometry -> LoadSurfaces(ist);
-	    return geometry;
-	  }
-	// else
-	// ist.putback (auxstring);
-      }
-    return NULL;
+	if (token != "csgsurfaces")
+        return nullptr;
+
+    CSGeometry * geometry = new CSGeometry ("");
+	geometry -> LoadSurfaces(ist);
+	return geometry;
   }
 
 

libsrc/csg/csgeom.hpp

@@ -18,7 +18,6 @@ namespace netgen
   class TriangleApproximation;
   class TATriangle;
 
-
   /**
      A top level object is an entity to be meshed.
      I can be either a solid, or one surface patch of a solid.
@@ -115,9 +114,9 @@ namespace netgen
     SymbolTable<Solid*> solids;
 
     /// all 2d splinecurves
-    SymbolTable< SplineGeometry<2>* > splinecurves2d;
+    SymbolTable<shared_ptr<SplineGeometry<2>>> splinecurves2d;
     /// all 3d splinecurves
-    SymbolTable< SplineGeometry<3>* > splinecurves3d;
+    SymbolTable<shared_ptr<SplineGeometry<3>>> splinecurves3d;
 
     /// all top level objects: solids and surfaces
     NgArray<TopLevelObject*> toplevelobjects;
@@ -131,7 +130,7 @@ namespace netgen
     public:
       UserPoint() = default;
       UserPoint (Point<3> p, int _index) : Point<3>(p), index(_index) { ; }
-      UserPoint (Point<3> p, const string & _name) : Point<3>(p), name(_name), index(-1) { ; } 
+      UserPoint (Point<3> p, const string & _name) : Point<3>(p), index(-1), name(_name) { ; } 
       int GetIndex() const { return index; }
       const string & GetName() const { return name; } 
       void DoArchive(Archive& archive)
@@ -182,7 +181,7 @@ namespace netgen
 
     void Clean ();
 
-    virtual void Save (string filename) const override;
+    virtual void Save (const filesystem::path & filename) const override;
     void Save (ostream & ost) const;
     void Load (istream & ist);
 
@@ -232,10 +231,10 @@ namespace netgen
     const SymbolTable<Solid*> & GetSolids () const { return solids; }
 
 
-    void SetSplineCurve (const char * name, SplineGeometry<2> * spl);
-    void SetSplineCurve (const char * name, SplineGeometry<3> * spl);
-    const SplineGeometry<2> * GetSplineCurve2d (const string & name) const;
-    const SplineGeometry<3> * GetSplineCurve3d (const string & name) const;
+    void SetSplineCurve (const char * name, shared_ptr<SplineGeometry<2>> spl);
+    void SetSplineCurve (const char * name, shared_ptr<SplineGeometry<3>> spl);
+    shared_ptr<SplineGeometry<2>> GetSplineCurve2d (const string & name) const;
+    shared_ptr<SplineGeometry<3>> GetSplineCurve3d (const string & name) const;
 
     void DoArchive(Archive& archive) override;
     

libsrc/csg/csgparser.cpp

@@ -479,7 +479,7 @@ namespace netgen
 		}
 
 	      Primitive * nprim = new Revolution(p0,p1,
-						 *(geom->GetSplineCurve2d(spline)));
+						 geom->GetSplineCurve2d(spline));
 
 	      geom->AddSurfaces (nprim);
 	      return new Solid(nprim);
@@ -511,8 +511,8 @@ namespace netgen
 		  break;
 		}
 	      
-	      Primitive * nprim = new Extrusion(*(geom->GetSplineCurve3d(epath)),
-						*(geom->GetSplineCurve2d(profile)),
+	      Primitive * nprim = new Extrusion(geom->GetSplineCurve3d(epath),
+						geom->GetSplineCurve2d(profile),
 						z_dir);
 	      geom->AddSurfaces (nprim);
 	      return new Solid(nprim);
@@ -1186,7 +1186,7 @@ namespace netgen
 		ParseChar (scan, '=');
 		ParseChar (scan, '(');
 		
-		SplineGeometry<2> * newspline = new SplineGeometry<2>;
+		auto newspline = make_shared<SplineGeometry<2>>();
 		// newspline->CSGLoad(scan);
 		LoadSpline (*newspline, scan);
 
@@ -1212,7 +1212,7 @@ namespace netgen
 		ParseChar (scan, '=');
 		ParseChar (scan, '(');
 		
-		SplineGeometry<3> * newspline = new SplineGeometry<3>;
+		auto newspline = make_shared<SplineGeometry<3>>();
 		// newspline->CSGLoad(scan);
 		LoadSpline (*newspline, scan);
 

libsrc/csg/csgpkg.cpp

@@ -17,7 +17,6 @@ extern "C" int Ng_CSG_Init (Tcl_Interp * interp);
 
 namespace netgen
 {
-  // extern DLL_HEADER NetgenGeometry * ng_geometry;
   extern DLL_HEADER shared_ptr<NetgenGeometry> ng_geometry;
   extern DLL_HEADER shared_ptr<Mesh> mesh;
 
@@ -66,18 +65,18 @@ namespace netgen
 	    Point3d pmin = geometry->BoundingBox ().PMin();
 	    Point3d pmax = geometry->BoundingBox ().PMax();
 	    
-	    sprintf (buf, "%5.1lf", pmin.X());
+	    snprintf (buf, size(buf), "%5.1lf", pmin.X());
 	    Tcl_SetVar (interp, "::geooptions.minx", buf, 0);
-	    sprintf (buf, "%5.1lf", pmin.Y());
+	    snprintf (buf, size(buf), "%5.1lf", pmin.Y());
 	    Tcl_SetVar (interp, "::geooptions.miny", buf, 0);
-	    sprintf (buf, "%5.1lf", pmin.Z());
+	    snprintf (buf, size(buf), "%5.1lf", pmin.Z());
 	    Tcl_SetVar (interp, "::geooptions.minz", buf, 0);
 	    
-	    sprintf (buf, "%5.1lf", pmax.X());
+	    snprintf (buf, size(buf), "%5.1lf", pmax.X());
 	    Tcl_SetVar (interp, "::geooptions.maxx", buf, 0);
-	    sprintf (buf, "%5.1lf", pmax.Y());
+	    snprintf (buf, size(buf), "%5.1lf", pmax.Y());
 	    Tcl_SetVar (interp, "::geooptions.maxy", buf, 0);
-	    sprintf (buf, "%5.1lf", pmax.Z());
+	    snprintf (buf, size(buf), "%5.1lf", pmax.Z());
 	    Tcl_SetVar (interp, "::geooptions.maxz", buf, 0);
 	  }
       }
@@ -433,17 +432,17 @@ namespace netgen
 	if (!tlo) return TCL_OK;
 
 	char varname[50];
-	sprintf (varname, "%s(red)", propvar);
+	snprintf (varname, size(varname), "%s(red)", propvar);
 	double red = atof (Tcl_GetVar (interp, varname, 0));
-	sprintf (varname, "%s(blue)", propvar);
+	snprintf (varname, size(varname), "%s(blue)", propvar);
 	double blue = atof (Tcl_GetVar (interp, varname, 0));
-	sprintf (varname, "%s(green)", propvar);
+	snprintf (varname, size(varname), "%s(green)", propvar);
 	double green = atof (Tcl_GetVar (interp, varname, 0));
 	tlo -> SetRGB (red, green, blue);
 
-	sprintf (varname, "%s(visible)", propvar);
+	snprintf (varname, size(varname), "%s(visible)", propvar);
 	tlo -> SetVisible (bool(atoi (Tcl_GetVar (interp, varname, 0))));
-	sprintf (varname, "%s(transp)", propvar);
+	snprintf (varname, size(varname), "%s(transp)", propvar);
 	tlo -> SetTransparent (bool(atoi (Tcl_GetVar (interp, varname, 0))));
       }
 
@@ -461,24 +460,24 @@ namespace netgen
 
 	char varname[50], varval[10];
 
-	sprintf (varname, "%s(red)", propvar);
-	sprintf (varval, "%lf", tlo->GetRed());
+	snprintf (varname, size(varname), "%s(red)", propvar);
+	snprintf (varval, size(varval), "%lf", tlo->GetRed());
 	Tcl_SetVar (interp, varname, varval, 0);
 
-	sprintf (varname, "%s(green)", propvar);
-	sprintf (varval, "%lf", tlo->GetGreen());
+	snprintf (varname, size(varname), "%s(green)", propvar);
+	snprintf (varval, size(varval), "%lf", tlo->GetGreen());
 	Tcl_SetVar (interp, varname, varval, 0);
 
-	sprintf (varname, "%s(blue)", propvar);
-	sprintf (varval, "%lf", tlo->GetBlue());
+	snprintf (varname, size(varname), "%s(blue)", propvar);
+	snprintf (varval, size(varval), "%lf", tlo->GetBlue());
 	Tcl_SetVar (interp, varname, varval, 0);
 
-	sprintf (varname, "%s(visible)", propvar);
-	sprintf (varval, "%d", tlo->GetVisible());
+	snprintf (varname, size(varname), "%s(visible)", propvar);
+	snprintf (varval, size(varval), "%d", tlo->GetVisible());
 	Tcl_SetVar (interp, varname, varval, 0);
 
-	sprintf (varname, "%s(transp)", propvar);
-	sprintf (varval, "%d", tlo->GetTransparent());
+	snprintf (varname, size(varname), "%s(transp)", propvar);
+	snprintf (varval, size(varval), "%d", tlo->GetTransparent());
 	Tcl_SetVar (interp, varname, varval, 0);
       }
 
@@ -547,86 +546,10 @@ namespace netgen
   }
 
 
-  /*
-  class CSGeometryRegister : public GeometryRegister
-  {
-  public:
-    virtual NetgenGeometry * Load (string filename) const;
-    virtual NetgenGeometry * LoadFromMeshFile (istream & ist) const;
-    virtual VisualScene * GetVisualScene (const NetgenGeometry * geom) const;
-  };
-
-  extern CSGeometry * ParseCSG (istream & istr);
-
-  NetgenGeometry *  CSGeometryRegister :: Load (string filename) const
-  {
-    const char * cfilename = filename.c_str();
-    if (strcmp (&cfilename[strlen(cfilename)-3], "geo") == 0)
-      {
-	PrintMessage (1, "Load CSG geometry file ", cfilename);
-
-	ifstream infile(cfilename);
-
-	CSGeometry * hgeom = ParseCSG (infile);
-	if (!hgeom)
-	  throw NgException ("geo-file should start with 'algebraic3d'");
-
-	hgeom -> FindIdenticSurfaces(1e-8 * hgeom->MaxSize()); 
-	return hgeom;
-      }
-
-    if (strcmp (&cfilename[strlen(cfilename)-3], "ngg") == 0)
-      {
-	PrintMessage (1, "Load new CSG geometry file ", cfilename);
-
-	ifstream infile(cfilename);
-	CSGeometry * hgeom = new CSGeometry("");
-	hgeom -> Load (infile);
-
-	return hgeom;
-      }
-
-
-    
-    return NULL;
-  }
-
-  NetgenGeometry * CSGeometryRegister :: LoadFromMeshFile (istream & ist) const 
-  {
-    string auxstring;
-    if (ist.good())
-      {
-	ist >> auxstring;
-	if (auxstring == "csgsurfaces")
-	  {
-	    CSGeometry * geometry = new CSGeometry ("");
-	    geometry -> LoadSurfaces(ist);
-	    return geometry;
-	  }
-	// else
-	// ist.putback (auxstring);
-      }
-    return NULL;
-  }
-
-  VisualScene * CSGeometryRegister :: GetVisualScene (const NetgenGeometry * geom) const
-  {
-    CSGeometry * geometry = dynamic_cast<CSGeometry*> (ng_geometry.get());
-    if (geometry)
-      {
-	vsgeom.SetGeometry (geometry);
-	return &vsgeom;
-      }
-    return NULL;
-  }
-  */
-
-
-
   class CSGeometryVisRegister : public GeometryRegister
   {
   public:
-    virtual NetgenGeometry * Load (string filename) const { return NULL; }
+    virtual NetgenGeometry * Load (const filesystem::path & filename) const { return NULL; }
     virtual VisualScene * GetVisualScene (const NetgenGeometry * geom) const;
   };
 

libsrc/csg/curve2d.hpp

@@ -13,7 +13,7 @@ namespace netgen
 
   /*
 
-  2D Curve repesentation
+  2D Curve representation
 
   */
 

libsrc/csg/edgeflw.cpp

@@ -180,7 +180,7 @@ namespace netgen
 
 	pi1 = 0;
 	copyedge = 0;
-	// identifyable point available ?
+	// identifiable point available ?
 
 	
 	for (int i = 0; i < geometry.identifications.Size() && !pi1; i++)
@@ -191,8 +191,8 @@ namespace netgen
 			 << ", v = " << specpoints[startpoints[j]].v 
 			 << " for copying (i,j = " << i << ", " << j << ")" << endl;	  
 #endif
- 	      if (geometry.identifications[i]->IdentifyableCandidate (specpoints[startpoints[j]]) &&
-		  geometry.identifications[i]->IdentifyableCandidate (specpoints[endpoints[j]]))
+ 	      if (geometry.identifications[i]->IdentifiableCandidate (specpoints[startpoints[j]]) &&
+		  geometry.identifications[i]->IdentifiableCandidate (specpoints[endpoints[j]]))
 		  
 	      
 		{
@@ -201,7 +201,7 @@ namespace netgen
 		
 		  for (int k = 0; k < hsp.Size() && !pi1; k++)
 		    {
-		      //(*testout) << "   ? identifyable with " << specpoints[hsp[k]].p 
+		      //(*testout) << "   ? identifiable with " << specpoints[hsp[k]].p 
 		      //<< ", v = " << specpoints[hsp[k]].v
 		      //		 << endl;
 		      if (identification_used.Used (INDEX_2(i, startpoints[j])) ||
@@ -212,12 +212,12 @@ namespace netgen
 			}
 		      
 		      if (geometry.identifications[i]
-			  ->Identifyable(specpoints[startpoints[j]], specpoints[hsp[k]], specpoint2tlo, specpoint2surface) ||
+			  ->Identifiable(specpoints[startpoints[j]], specpoints[hsp[k]], specpoint2tlo, specpoint2surface) ||
 			  geometry.identifications[i]
-			  ->Identifyable(specpoints[hsp[k]], specpoints[startpoints[j]], specpoint2tlo, specpoint2surface))
+			  ->Identifiable(specpoints[hsp[k]], specpoints[startpoints[j]], specpoint2tlo, specpoint2surface))
 			{
 #ifdef DEVELOP
-			  (*testout) << "identifyable: " << specpoints[hsp[k]].p << ", v = " << specpoints[hsp[k]].v
+			  (*testout) << "identifiable: " << specpoints[hsp[k]].p << ", v = " << specpoints[hsp[k]].v
 				     << " and " << specpoints[startpoints[j]].p << ", v = " << specpoints[startpoints[j]].v 
 				     << " (identification " << i+1 << ")" << endl;
 #endif
@@ -245,7 +245,7 @@ namespace netgen
 	    }
 	
       
-	// cannot copy from other ege ?
+	// cannot copy from other edge ?
 	if (!pi1)
 	  checkedcopy = startpoints.Size();
       
@@ -567,7 +567,7 @@ namespace netgen
     {
     //    int i, j;
     SegmentIndex si;
-    PointIndex pi;
+    // PointIndex pi;
 
     NgArray<int> osedges(cntedge);
     INDEX_2_HASHTABLE<int> osedgesht (cntedge+1);
@@ -610,7 +610,7 @@ namespace netgen
     for (int i = 1; i <= osedgesht.GetNBags(); i++)
       for (int j = 1; j <= osedgesht.GetBagSize(i); j++)
 	{
-	  INDEX_2 i2; 
+	  PointIndices<2> i2; 
 	  int val;
 	  osedgesht.GetData (i, j, i2, val);
 
@@ -619,8 +619,8 @@ namespace netgen
 	  Vec<3> v = p2 - p1;
 	  double vlen = v.Length();
 	  v /= vlen;
-	  for (pi = PointIndex::BASE; 
-	       pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	  for (PointIndex pi = IndexBASE<PointIndex>(); 
+	       pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 
 	    if (pi != i2.I1() && pi != i2.I2())
 	      {
@@ -970,7 +970,7 @@ namespace netgen
   
     for (int i = 0; i < geometry.GetNTopLevelObjects(); i++)
       {
-	Solid * locsol;
+	// Solid * locsol;
 
 	if (geometry.GetTopLevelObject(i)->GetLayer() != layer) 
 	  continue;
@@ -978,7 +978,8 @@ namespace netgen
 	const Solid * sol = geometry.GetTopLevelObject(i)->GetSolid();
 	const Surface * surf = geometry.GetTopLevelObject(i)->GetSurface();
 
-	sol -> TangentialSolid (hp, locsol, locsurfind, size*ideps);
+	// sol -> TangentialSolid (hp, locsol, locsurfind, size*ideps);
+        auto locsol = sol -> TangentialSolid (hp, locsurfind, size*ideps);
 
 	//*testout << "hp = " << hp << endl;
 	//(*testout) << "locsol: " << endl;
@@ -995,7 +996,8 @@ namespace netgen
 	ReducePrimitiveIterator rpi(boxp);
 	UnReducePrimitiveIterator urpi;
       
-	((Solid*)locsol) -> IterateSolid (rpi);
+	// ((Solid*)locsol) -> IterateSolid (rpi);
+        locsol -> IterateSolid (rpi);
 
 	locsol -> CalcSurfaceInverse ();
       
@@ -1020,7 +1022,8 @@ namespace netgen
 		}
 	  }
 
-	((Solid*)locsol) -> IterateSolid (urpi);
+	// ((Solid*)locsol) -> IterateSolid (urpi);
+        locsol -> IterateSolid (urpi);
 
       
 	if (debug)
@@ -1085,23 +1088,33 @@ namespace netgen
 	    //int k;
 	    double eps = 1e-8*size;
 
-	    NgArray<bool> pre_ok(2);
+            ArrayMem<bool,2> pre_ok(2);
+            bool flip = false;
 
  	    do
  	      {
  		eps *= 0.5;
- 		pre_ok[0] = (locsol -> VectorIn2 (hp, m, n, eps) == IS_OUTSIDE &&
-			     locsol -> VectorIn2 (hp, m, -1. * n, eps) == IS_INSIDE);
- 		pre_ok[1] = (locsol -> VectorIn2 (hp, -1.*m, n, eps) == IS_OUTSIDE &&
-			     locsol -> VectorIn2 (hp, -1.*m, -1. * n, eps) == IS_INSIDE);
+                auto in00 = locsol -> VectorIn2 (hp, m, n, eps);
+                auto in01 = locsol -> VectorIn2 (hp, m, -1. * n, eps);
+                pre_ok[0] = in00 == IS_OUTSIDE && in01 == IS_INSIDE;
+
+                if(in00 == IS_INSIDE && in01 == IS_OUTSIDE)
+                  pre_ok[0] = flip = true;
+
+                auto in10 = locsol -> VectorIn2 (hp, -1.*m, n, eps);
+                auto in11 = locsol -> VectorIn2 (hp, -1.*m, -1. * n, eps);
+                pre_ok[1] = (in10 == IS_OUTSIDE && in11 == IS_INSIDE);
+
+                if(in10 == IS_INSIDE && in11 == IS_OUTSIDE)
+                  pre_ok[1] = flip = true;
 		
 		if (debug)
 		  {
 		    *testout << "eps = " << eps << endl;
-		    *testout << "in,1 = " << locsol -> VectorIn2 (hp, m, n, eps) << endl;
-		    *testout << "in,1 = " << locsol -> VectorIn2 (hp, m, -1. * n, eps) << endl;
-		    *testout << "in,1 = " << locsol -> VectorIn2 (hp, -1.*m, n, eps) << endl;
-		    *testout << "in,1 = " << locsol -> VectorIn2 (hp, -1.*m, -1. * n, eps) << endl;
+                    *testout << "in,1 = " << in00 << endl;
+                    *testout << "in,1 = " << in01 << endl;
+                    *testout << "in,1 = " << in10 << endl;
+                    *testout << "in,1 = " << in11 << endl;
 		  }
  	      }
  	    while(pre_ok[0] && pre_ok[1] && eps > 1e-16*size);
@@ -1150,10 +1163,10 @@ namespace netgen
 		m2 = fac * grad;
 		// (*testout) << "hp = " << hp << ", m = " << m << ", m2 = " << m2 << endl;
 
-		Solid * locsol2;
-		locsol -> TangentialSolid3 (hp, m, m2, locsol2, locsurfind2, ideps*size);
+		// Solid * locsol2;
+		auto locsol2 = locsol -> TangentialSolid3 (hp, m, m2, locsurfind2, ideps*size);
 		if (!locsol2) ok = 0;
-		delete locsol2;
+		// delete locsol2;
 
 
 		if (ok)
@@ -1197,7 +1210,10 @@ namespace netgen
 		  
 		    if (!surf)
 		      {
-			if (sameasref)
+                        bool inside = sameasref;
+                        if(flip)
+                          inside = !inside;
+                        if (inside)
 			  refedges.Elem(hi).domin = i;
 			else 
 			  refedges.Elem(hi).domout = i;
@@ -1246,7 +1262,7 @@ namespace netgen
 		m *= -1;
 	      } 
 	  }
-	delete locsol;          
+	// delete locsol;          
       }
 
    
@@ -1355,8 +1371,8 @@ namespace netgen
     lastpi = PointIndex::INVALID;
 
     /*
-    for (pi = PointIndex::BASE; 
-	 pi < mesh.GetNP()+PointIndex::BASE; pi++)
+    for (pi = IndexBASE<PointIndex>(); 
+	 pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
       if (Dist (mesh[pi], p) < 1e-6)
 	{
 	  lastpi = pi;
@@ -1398,8 +1414,8 @@ namespace netgen
 	if (i == ne)
 	  {
 	    /*
-	  for (pi = PointIndex::BASE; 
-	       pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	  for (pi = IndexBASE<PointIndex>(); 
+	       pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 	    if (Dist(mesh[pi], np) < 1e-6)
 	      thispi = pi;
 	    */
@@ -1523,8 +1539,8 @@ namespace netgen
     // generate initial point
     Point<3> p = edgepoints[0];
     PointIndex pi1 = PointIndex::INVALID;
-    for (pi = PointIndex::BASE; 
-	 pi < mesh.GetNP()+PointIndex::BASE; pi++)
+    for (PointIndex pi = IndexBASE<PointIndex>(); 
+	 pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 
       if (Dist (mesh[pi], p) < 1e-6*geometry.MaxSize())
 	{
@@ -1541,8 +1557,8 @@ namespace netgen
 
     p = edgepoints.Last();
     PointIndex pi2 = PointIndex::INVALID;
-    for (pi = PointIndex::BASE; 
-	 pi < mesh.GetNP()+PointIndex::BASE; pi++)
+    for (pi = IndexBASE<PointIndex>(); 
+	 pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 
       if (Dist (mesh[pi], p) < 1e-6*geometry.MaxSize())
 	{
@@ -1630,7 +1646,7 @@ namespace netgen
 	    Mesh & mesh)
   {
     int k;
-    PointIndex pi;
+    // PointIndex pi;
 
     double size = geometry.MaxSize();
     
@@ -1644,8 +1660,8 @@ namespace netgen
       
 	PointIndex frompi = PointIndex::INVALID;
 	PointIndex topi = PointIndex::INVALID;
-	for (pi = PointIndex::BASE; 
-	     pi < mesh.GetNP()+PointIndex::BASE; pi++)
+	for (PointIndex pi = IndexBASE<PointIndex>(); 
+	     pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
 	  {
 	    if (Dist2 (mesh[pi], fromp) <= 1e-16*size)
 	      frompi = pi;
@@ -1664,9 +1680,9 @@ namespace netgen
 	  (*geometry.identifications.Get(copyedgeidentification));
 
 
-	if (csi.Identifyable (mesh[frompi], mesh[topi]))
+	if (csi.Identifiable (mesh[frompi], mesh[topi]))
 	  mesh.GetIdentifications().Add(frompi, topi, copyedgeidentification);
-	else if (csi.Identifyable (mesh[topi], mesh[frompi]))
+	else if (csi.Identifiable (mesh[topi], mesh[frompi]))
 	  mesh.GetIdentifications().Add(topi, frompi, copyedgeidentification);
 	else
 	  {
@@ -1698,12 +1714,12 @@ namespace netgen
 	if (oldseg.seginfo == 0)
 	  continue;
 
-	int pi1 = oldseg[0];
-	int pi2 = oldseg[1];
+	PointIndex pi1 = oldseg[0];
+	PointIndex pi2 = oldseg[1];
 
-	int npi1 = geometry.identifications.Get(copyedgeidentification)
+	PointIndex npi1 = geometry.identifications.Get(copyedgeidentification)
 	  -> GetIdentifiedPoint (mesh, pi1);
-	int npi2 = geometry.identifications.Get(copyedgeidentification)
+	PointIndex npi2 = geometry.identifications.Get(copyedgeidentification)
 	  -> GetIdentifiedPoint (mesh, pi2);
 
 	//(*testout) << "copy edge, pts = " << npi1 << " - " << npi2 << endl;
@@ -1767,7 +1783,7 @@ namespace netgen
     int nsurf = geometry.GetNSurf();
     int layer = 0;
 
-    Solid * tansol;
+    // Solid * tansol;
     NgArray<int> tansurfind;
 
     double size = geometry.MaxSize();
@@ -1825,7 +1841,8 @@ namespace netgen
 		  continue;
 		  
 		const Solid * sol = geometry.GetTopLevelObject(j)->GetSolid();
-		sol -> TangentialSolid (p1, tansol, tansurfind, ideps*size);
+		// sol -> TangentialSolid (p1, tansol, tansurfind, ideps*size);
+                auto tansol = sol -> TangentialSolid (p1, tansurfind, ideps*size);
 		layer = geometry.GetTopLevelObject(j)->GetLayer();
 
 		
@@ -1855,7 +1872,7 @@ namespace netgen
 			//        seg.invs1 = surfaces[i] -> Inverse();
 			//        seg.invs2 = ! (surfaces[i] -> Inverse());
 		      }
-		    delete tansol;
+		    // delete tansol;
 		  }
 	      }
 
@@ -1868,12 +1885,10 @@ namespace netgen
 
 	    if (seg1.domin != -1 || seg1.domout != -1)
 	      {
-		mesh.AddPoint (p1, layer, EDGEPOINT);
-		mesh.AddPoint (p2, layer, EDGEPOINT);
-		seg1[0] = mesh.GetNP()-1;
-		seg1[1] = mesh.GetNP();
-		seg2[1] = mesh.GetNP()-1;
-		seg2[0] = mesh.GetNP();
+                seg1[0] = mesh.AddPoint (p1, layer, EDGEPOINT);
+                seg1[1] = mesh.AddPoint (p2, layer, EDGEPOINT);
+                seg2[0] = seg1[1];
+                seg2[1] = seg1[0];
 		seg1.geominfo[0].trignum = 1;
 		seg1.geominfo[1].trignum = 1;
 		seg2.geominfo[0].trignum = 1;

libsrc/csg/explicitcurve2d.hpp

@@ -13,7 +13,7 @@ namespace netgen
 
   /*
 
-  Explicit 2D Curve repesentation
+  Explicit 2D Curve representation
 
   */
 

libsrc/csg/extrusion.cpp

@@ -1,4 +1,5 @@
 #include <mystdlib.h>
+#include <core/register_archive.hpp>
 
 #include <linalg.hpp>
 #include <csg.hpp>
@@ -41,6 +42,18 @@ namespace netgen
 	    loc_z_dir[i] = glob_z_direction;
 	  }
       }
+
+    double cum_angle = 0.;
+    for(auto i : Range(path->GetSplines()))
+      {
+        const auto& sp = path->GetSpline(i);
+        auto t1 = sp.GetTangent(0.);
+        t1.Normalize();
+        auto t2 = sp.GetTangent(1.);
+        t2.Normalize();
+        cum_angle += acos(t1 * t2);
+        angles.Append(cum_angle);
+      }
     
     profile->GetCoeff(profile_spline_coeff);
     latest_point3d = -1.111e30;
@@ -128,16 +141,21 @@ namespace netgen
   void ExtrusionFace :: CalcProj(const Point<3> & point3d, Point<2> & point2d,
 				 int & seg, double & t) const
   {
-    if (Dist2 (point3d, latest_point3d) < 
-        1e-25 * Dist2(path->GetSpline(0).StartPI(), path->GetSpline(0).EndPI()))
+    static mutex set_latest_point;
+
+    auto eps = 1e-25 * Dist2(path->GetSpline(0).StartPI(), path->GetSpline(0).EndPI());
+    if (Dist2 (point3d, latest_point3d) < eps)
       {
-	point2d = latest_point2d;
-	seg = latest_seg;
-	t = latest_t;
-	return;
+        std::lock_guard<std::mutex> guard(set_latest_point);
+        if (Dist2 (point3d, latest_point3d) < eps)
+        {
+            point2d = latest_point2d;
+            seg = latest_seg;
+            t = latest_t;
+            return;
+        }
       }
     
-    latest_point3d = point3d;
 
     double cutdist = -1;
     
@@ -201,11 +219,13 @@ namespace netgen
 	    point2d = testpoint2d;
 	    t = thist;
 	    seg = i;
-	    latest_seg = i;
-	    latest_t = t;
-	    latest_point2d = point2d;
 	  }
       }
+    std::lock_guard<std::mutex> guard(set_latest_point);
+    latest_seg = seg;
+    latest_t = t;
+    latest_point2d = point2d;
+    latest_point3d = point3d;
   }
 
   double ExtrusionFace :: CalcProj(const Point<3> & point3d, Point<2> & point2d,
@@ -415,6 +435,14 @@ namespace netgen
   }
 
 
+  bool ExtrusionFace :: PointInFace (const Point<3> & p, const double eps) const
+  {
+    Point<3> hp = p;
+    Project(hp);
+    return Dist2(p,hp) < sqr(eps);
+  }
+  
+
   void ExtrusionFace :: LineIntersections ( const Point<3> & p,
 					    const Vec<3> & v,
 					    const double eps,
@@ -648,20 +676,35 @@ namespace netgen
     dez /= lenz;
     dez -= (dez * ez) * ez;
   }
-  
 
-  Extrusion :: Extrusion(const SplineGeometry<3> & path_in,
-			 const SplineGeometry<2> & profile_in,
+  void ExtrusionFace :: DefineTangentialPlane(const Point<3>& ap1,
+                                              const Point<3>& ap2)
+  {
+    Surface::DefineTangentialPlane(ap1, ap2);
+    tangential_plane_seg = latest_seg;
+  }
+
+  void ExtrusionFace :: ToPlane(const Point<3>& p3d, Point<2>& p2d,
+                                double h, int& zone) const
+  {
+    Surface::ToPlane(p3d, p2d, h, zone);
+    double angle = angles[tangential_plane_seg] - angles[latest_seg];
+    if(fabs(angle) > 3.14/2.)
+      zone = -1;
+  }
+
+  Extrusion :: Extrusion(shared_ptr<SplineGeometry<3>> path_in,
+			 shared_ptr<SplineGeometry<2>> profile_in,
 			 const Vec<3> & z_dir) :
-    path(&path_in), profile(&profile_in), z_direction(z_dir)
+    path(path_in), profile(profile_in), z_direction(z_dir)
   {
     surfaceactive.SetSize(0);
     surfaceids.SetSize(0);
 
     for(int j=0; j<profile->GetNSplines(); j++)
       {
-	ExtrusionFace * face = new ExtrusionFace(&((*profile).GetSpline(j)),
-						 path,
+	ExtrusionFace * face = new ExtrusionFace(&(profile->GetSpline(j)),
+						 path.get(),
 						 z_direction);
 	faces.Append(face);
 	surfaceactive.Append(true);
@@ -737,6 +780,16 @@ namespace netgen
     return PointInSolid(p,eps,NULL);    
   }
 
+  void Extrusion :: GetTangentialSurfaceIndices (const Point<3> & p, 
+                                                 NgArray<int> & surfind, double eps) const
+  {
+    for (int j = 0; j < faces.Size(); j++)
+      if (faces[j] -> PointInFace(p, eps))
+        if (!surfind.Contains (GetSurfaceId(j)))
+          surfind.Append (GetSurfaceId(j));
+  }
+
+  
   INSOLID_TYPE Extrusion :: VecInSolid (const Point<3> & p,
 					const Vec<3> & v,
 					double eps) const
@@ -838,7 +891,7 @@ namespace netgen
       return retval;
 
     if(latestfacenum >= 0)
-      return faces[latestfacenum]->VecInFace(p,v2,0);
+      return faces[latestfacenum]->VecInFace(p,v2,eps);
     else
       return VecInSolid(p,v2,eps);
   }

libsrc/csg/extrusion.hpp

@@ -12,8 +12,10 @@ namespace netgen
     const SplineSeg<2> * profile;
     const SplineGeometry<3> * path;
     Vec<3> glob_z_direction;
+    Array<double> angles;
 
     bool deletable;
+    int tangential_plane_seg;
   
     NgArray< const SplineSeg3<3> * > spline3_path;
     NgArray< const LineSeg<3> * > line_path;
@@ -54,7 +56,7 @@ namespace netgen
 
     ~ExtrusionFace();
 
-    virtual void DoArchive(Archive& ar)
+    void DoArchive(Archive& ar) override
     {
       Surface::DoArchive(ar);
       ar & profile & path & glob_z_direction & deletable & spline3_path & line_path &
@@ -62,25 +64,25 @@ namespace netgen
         profile_spline_coeff & latest_seg & latest_t & latest_point2d & latest_point3d;
     }
   
-    virtual int IsIdentic (const Surface & s2, int & inv, double eps) const; 
+    int IsIdentic (const Surface & s2, int & inv, double eps) const override;
   
-    virtual double CalcFunctionValue (const Point<3> & point) const;
-    virtual void CalcGradient (const Point<3> & point, Vec<3> & grad) const;
-    virtual void CalcHesse (const Point<3> & point, Mat<3> & hesse) const;
-    virtual double HesseNorm () const;
+    double CalcFunctionValue (const Point<3> & point) const override;
+    void CalcGradient (const Point<3> & point, Vec<3> & grad) const override;
+    void CalcHesse (const Point<3> & point, Mat<3> & hesse) const override;
+    double HesseNorm () const override;
 
-    virtual double MaxCurvature () const;
+    double MaxCurvature () const override;
     //virtual double MaxCurvatureLoc (const Point<3> & /* c */ , 
     //				  double /* rad */) const;
 
-    virtual void Project (Point<3> & p) const;
+    void Project (Point<3> & p) const override;
 
-    virtual Point<3> GetSurfacePoint () const;
-    virtual void Print (ostream & str) const;
+    Point<3> GetSurfacePoint () const override;
+    void Print (ostream & str) const override;
   
-    virtual void GetTriangleApproximation (TriangleApproximation & tas, 
+    void GetTriangleApproximation (TriangleApproximation & tas, 
 					   const Box<3> & boundingbox, 
-					   double facets) const;
+					   double facets) const override;
 
     const SplineGeometry<3> & GetPath(void) const {return *path;}
     const SplineSeg<2> & GetProfile(void) const {return *profile;}
@@ -94,6 +96,9 @@ namespace netgen
 			     int & after,
 			     bool & intersecting ) const;
 
+
+    bool PointInFace (const Point<3> & p, const double eps) const;
+    
     INSOLID_TYPE VecInFace ( const Point<3> & p,
 			     const Vec<3> & v,
 			     const double eps ) const;
@@ -111,6 +116,11 @@ namespace netgen
 				    Vec<3> & ex, Vec<3> & ey, Vec<3> & ez,
 				    Vec<3> & dex, Vec<3> & dey, Vec<3> & dez) const;
 
+    void DefineTangentialPlane(const Point<3>& ap1,
+                               const Point<3>& ap2) override;
+    void ToPlane(const Point<3>& p3d, Point<2>& p2d,
+                 double h, int& zone) const override;
+
   };
 
 
@@ -118,8 +128,8 @@ namespace netgen
   class Extrusion : public Primitive
   {
   private:
-    const SplineGeometry<3>* path;
-    const SplineGeometry<2>* profile; // closed, clockwise oriented curve
+    shared_ptr<SplineGeometry<3>> path;
+    shared_ptr<SplineGeometry<2>> profile; // closed, clockwise oriented curve
 
     Vec<3> z_direction;
 
@@ -128,43 +138,46 @@ namespace netgen
     mutable int latestfacenum;
 
   public:
-    Extrusion(const SplineGeometry<3> & path_in,
-	      const SplineGeometry<2> & profile_in,
+    Extrusion(shared_ptr<SplineGeometry<3>> path_in,
+	      shared_ptr<SplineGeometry<2>> profile_in,
 	      const Vec<3> & z_dir);
     // default constructor for archive
     Extrusion() {}
     ~Extrusion();
 
-    virtual void DoArchive(Archive& ar)
+    void DoArchive(Archive& ar) override
     {
       Primitive::DoArchive(ar);
       ar & path & profile & z_direction & faces & latestfacenum;
     }
-    virtual INSOLID_TYPE BoxInSolid (const BoxSphere<3> & box) const;
-    virtual INSOLID_TYPE PointInSolid (const Point<3> & p,
-				       double eps) const;
+    INSOLID_TYPE BoxInSolid (const BoxSphere<3> & box) const override;
+    INSOLID_TYPE PointInSolid (const Point<3> & p,
+                               double eps) const override;
     INSOLID_TYPE PointInSolid (const Point<3> & p,
 			       double eps,
 			       NgArray<int> * const facenums) const;
-    virtual INSOLID_TYPE VecInSolid (const Point<3> & p,
-				     const Vec<3> & v,
-				     double eps) const;
+
+    void GetTangentialSurfaceIndices (const Point<3> & p, 
+                                      NgArray<int> & surfind, double eps) const override;
+
+    INSOLID_TYPE VecInSolid (const Point<3> & p,
+                             const Vec<3> & v,
+                             double eps) const override;
 
     // checks if lim s->0 lim t->0  p + t(v1 + s v2) in solid
-    virtual INSOLID_TYPE VecInSolid2 (const Point<3> & p,
-				      const Vec<3> & v1,
-				      const Vec<3> & v2,
-				      double eps) const;
+    INSOLID_TYPE VecInSolid2 (const Point<3> & p,
+                              const Vec<3> & v1,
+                              const Vec<3> & v2,
+                              double eps) const override;
 
   
-    virtual int GetNSurfaces() const;
-    virtual Surface & GetSurface (int i = 0);
-    virtual const Surface & GetSurface (int i = 0) const;
+    int GetNSurfaces() const override;
+    Surface & GetSurface (int i = 0) override;
+    const Surface & GetSurface (int i = 0) const override;
 
 
-    virtual void Reduce (const BoxSphere<3> & box);
-    virtual void UnReduce ();
-
+    void Reduce (const BoxSphere<3> & box) override;
+    void UnReduce () override;
   };
 
 }

libsrc/csg/genmesh.cpp

@@ -10,15 +10,19 @@
 
 namespace netgen
 {
-  NgArray<SpecialPoint> specpoints;
-  static NgArray<MeshPoint> spoints;
 
+  DLL_HEADER NgArray<SpecialPoint> global_specpoints;  // for visualization
+  //static NgArray<MeshPoint> spoints;
+  
 #define TCL_OK 0
 #define TCL_ERROR 1
 
 
 
-  static void FindPoints (CSGeometry & geom, Mesh & mesh)
+  static void FindPoints (CSGeometry & geom,
+                          NgArray<SpecialPoint> &  specpoints,
+                          NgArray<MeshPoint> & spoints,
+                          Mesh & mesh)
   {
     PrintMessage (1, "Start Findpoints");
 
@@ -31,7 +35,7 @@ namespace netgen
         auto up = geom.GetUserPoint(i);
 	auto pnum = mesh.AddPoint(up);
 	mesh.Points().Last().Singularity (geom.GetUserPointRefFactor(i));
-	mesh.AddLockedPoint (PointIndex (i+1));
+	mesh.AddLockedPoint (pnum);
         int index = up.GetIndex();
         if (index == -1)
           index = mesh.AddCD3Name (up.GetName())+1;
@@ -48,7 +52,13 @@ namespace netgen
     
     PrintMessage (2, "Analyze spec points");
     spc.AnalyzeSpecialPoints (geom, spoints, specpoints);
-  
+
+    {
+      static mutex mut;
+      lock_guard<mutex> guard(mut);
+      global_specpoints = specpoints;
+    }
+    
     PrintMessage (5, "done");
 
     (*testout) << specpoints.Size() << " special points:" << endl;
@@ -67,7 +77,10 @@ namespace netgen
 
 
 
-  static void FindEdges (CSGeometry & geom, Mesh & mesh, MeshingParameters & mparam,
+  static void FindEdges (CSGeometry & geom, Mesh & mesh,
+                         NgArray<SpecialPoint> &  specpoints,
+                         NgArray<MeshPoint> & spoints,
+                         MeshingParameters & mparam,
                          const bool setmeshsize = false)
   {
     EdgeCalculation ec (geom, specpoints, mparam);
@@ -430,7 +443,7 @@ namespace netgen
 	meshing.SetStartTime (starttime);
 
         double eps = 1e-8 * geom.MaxSize();
-	for (PointIndex pi = PointIndex::BASE; pi < noldp+PointIndex::BASE; pi++)
+	for (PointIndex pi = IndexBASE<PointIndex>(); pi < noldp+IndexBASE<PointIndex>(); pi++)
 	  { 
 	    // if(surf->PointOnSurface(mesh[pi]))
 	    meshing.AddPoint (mesh[pi], pi, NULL,
@@ -460,8 +473,8 @@ namespace netgen
 	  {
 	    PointGeomInfo gi;
 	    gi.trignum = k;
-	    meshing.AddBoundaryElement (segments[si][0] + 1 - PointIndex::BASE, 
-					segments[si][1] + 1 - PointIndex::BASE, 
+	    meshing.AddBoundaryElement (segments[si][0] + 1 - IndexBASE<PointIndex>(), 
+					segments[si][1] + 1 - IndexBASE<PointIndex>(), 
 					gi, gi);
 	  }
 
@@ -504,7 +517,7 @@ namespace netgen
 	  }
 
 	if (multithread.terminate) return;
-      
+        
 	for (SurfaceElementIndex sei = oldnf; sei < mesh.GetNSE(); sei++)
 	  mesh[sei].SetIndex (k);
 
@@ -669,6 +682,10 @@ namespace netgen
   int CSGGenerateMesh (CSGeometry & geom, 
 		       shared_ptr<Mesh> & mesh, MeshingParameters & mparam)
   {
+    NgArray<SpecialPoint> specpoints;
+    NgArray<MeshPoint> spoints;
+
+    
     if (mesh && mesh->GetNSE() &&
 	!geom.GetNSolids())
       {
@@ -701,11 +718,11 @@ namespace netgen
 
 	    mesh -> LoadLocalMeshSize (mparam.meshsizefilename);
             for (auto mspnt : mparam.meshsize_points)
-              mesh -> RestrictLocalH (mspnt.pnt, mspnt.h);
+              mesh -> RestrictLocalH (mspnt.pnt, mspnt.h, mspnt.layer);
 	  }
 
 	spoints.SetSize(0);
-	FindPoints (geom, *mesh);
+	FindPoints (geom, specpoints, spoints, *mesh);
       
 	PrintMessage (5, "find points done");
 
@@ -723,7 +740,7 @@ namespace netgen
 
     if (mparam.perfstepsstart <= MESHCONST_MESHEDGES)
       {
-	FindEdges (geom, *mesh, mparam, true);
+	FindEdges (geom, *mesh, specpoints, spoints, mparam, true);
 	if (multithread.terminate) return TCL_OK;
 #ifdef LOG_STREAM      
 	(*logout) << "Edges meshed" << endl
@@ -740,16 +757,16 @@ namespace netgen
 	    mesh->CalcLocalH(mparam.grading);
 	    mesh->DeleteMesh();
 	    
-	    FindPoints (geom, *mesh);
+	    FindPoints (geom, specpoints, spoints, *mesh);
 	    if (multithread.terminate) return TCL_OK;
-	    FindEdges (geom, *mesh, mparam, true);
+	    FindEdges (geom, *mesh, specpoints, spoints, mparam, true);
 	    if (multithread.terminate) return TCL_OK;
 	    
 	    mesh->DeleteMesh();
 	  
-	    FindPoints (geom, *mesh);
+	    FindPoints (geom, specpoints, spoints, *mesh);
 	    if (multithread.terminate) return TCL_OK;
-	    FindEdges (geom, *mesh, mparam);
+	    FindEdges (geom, *mesh, specpoints, spoints, mparam);
 	    if (multithread.terminate) return TCL_OK;
 	  }
       }
@@ -809,6 +826,9 @@ namespace netgen
       {
 	multithread.task = "Volume meshing";
 
+	for (int i = 0; i < geom.GetNTopLevelObjects(); i++)
+	  mesh->SetMaterial (i+1, geom.GetTopLevelObject(i)->GetMaterial().c_str());
+
 	MESHING3_RESULT res =
 	  MeshVolume (mparam, *mesh);
 
@@ -821,10 +841,6 @@ namespace netgen
 
 	MeshQuality3d (*mesh);
       
-	for (int i = 0; i < geom.GetNTopLevelObjects(); i++)
-	  mesh->SetMaterial (i+1, geom.GetTopLevelObject(i)->GetMaterial().c_str());
-      
-
 #ifdef STAT_STREAM
 	(*statout) << GetTime() << " & ";
 #endif      

libsrc/csg/identify.cpp

@@ -36,24 +36,24 @@ void Identification :: IdentifySpecialPoints (NgArray<class SpecialPoint> & poin
 
 
 int Identification :: 
-Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 	      const TABLE<int> & specpoint2solid,
 	      const TABLE<int> & specpoint2surface) const
 {
-  cout << "Identification::Identifyable called for base-class" << endl;
+  cout << "Identification::Identifiable called for base-class" << endl;
   return 0;
 }
 
 int Identification :: 
-Identifyable (const Point<3> & p1, const Point<3> & sp2) const
+Identifiable (const Point<3> & p1, const Point<3> & sp2) const
 {
-  cout << "Identification::Identifyable called for base-class" << endl;
+  cout << "Identification::Identifiable called for base-class" << endl;
   return 0;
 }
 
 
 int Identification :: 
-IdentifyableCandidate (const SpecialPoint & sp1) const
+IdentifiableCandidate (const SpecialPoint & sp1) const
 {
   return 1;
 }
@@ -65,10 +65,10 @@ ShortEdge (const SpecialPoint & sp1, const SpecialPoint & sp2) const
   return 0;
 }
 
-int Identification :: GetIdentifiedPoint (class Mesh & mesh, int pi)
+PointIndex Identification :: GetIdentifiedPoint (class Mesh & mesh, PointIndex pi)
 {
   cout << "Identification::GetIdentifiedPoint called for base-class" << endl;
-  return -1;
+  return PointIndex::INVALID;
 }
 
 void Identification :: IdentifyPoints (Mesh & mesh)
@@ -196,7 +196,7 @@ void PeriodicIdentification :: IdentifySpecialPoints
 */
 
 int PeriodicIdentification :: 
-Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 	      const TABLE<int> & specpoint2solid,
 	      const TABLE<int> & specpoint2surface) const
 {
@@ -252,7 +252,7 @@ Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 }
 
 int PeriodicIdentification :: 
-Identifyable (const Point<3> & p1, const Point<3> & p2) const
+Identifiable (const Point<3> & p1, const Point<3> & p2) const
 {
   return (s1->PointOnSurface (p1) &&
 	  s2->PointOnSurface (p2));
@@ -261,8 +261,8 @@ Identifyable (const Point<3> & p1, const Point<3> & p2) const
 
 
 
-int PeriodicIdentification :: 
-GetIdentifiedPoint (class Mesh & mesh,  int pi)
+PointIndex PeriodicIdentification :: 
+GetIdentifiedPoint (class Mesh & mesh, PointIndex pi)
 {
   const Surface *snew;
   const Point<3> & p = mesh.Point (pi);
@@ -289,14 +289,14 @@ GetIdentifiedPoint (class Mesh & mesh,  int pi)
   // project to other surface
   snew->Project (hp);
 
-  int newpi = 0;
-  for (int i = 1; i <= mesh.GetNP(); i++)
-    if (Dist2 (mesh.Point(i), hp) < 1e-12)
+  PointIndex newpi(PointIndex::INVALID);
+  for (PointIndex pi : Range(mesh.Points()))
+    if (Dist2 (mesh.Point(pi), hp) < 1e-12)
       {
-	newpi = i;
+	newpi = pi;
 	break;
       }
-  if (!newpi)
+  if (!newpi.IsValid())
     newpi = mesh.AddPoint (hp);
 
   if (snew == s2)
@@ -318,6 +318,11 @@ GetIdentifiedPoint (class Mesh & mesh,  int pi)
 
 void PeriodicIdentification :: IdentifyPoints (class Mesh & mesh)
 {
+  Point3d p1, p2;
+  mesh.GetBox(p1, p2);
+  auto eps = 1e-6 * (p2-p1).Length();
+
+  /*
   for (int i = 1; i <= mesh.GetNP(); i++)
     {
       Point<3> p = mesh.Point(i);
@@ -327,16 +332,27 @@ void PeriodicIdentification :: IdentifyPoints (class Mesh & mesh)
           pp = trafo(pp);
 	  s2->Project (pp);
 	  for (int j = 1; j <= mesh.GetNP(); j++)
-	    if (Dist2(mesh.Point(j), pp) < 1e-6)
+	    if (Dist2(mesh.Point(j), pp) < eps)
 	      {
 		mesh.GetIdentifications().Add (i, j, nr);
-		/*
-		(*testout) << "Identify points(periodic:), nr = " << nr << ": "
-			   << mesh.Point(i) << " - " << mesh.Point(j) << endl;
-		*/
 	      }
 	}
     }
+  */
+
+  for (auto pi : Range(mesh.Points()))
+    {
+      Point<3> p = mesh[pi];
+      if (s1->PointOnSurface (p))
+	{
+	  Point<3> pp = p;
+          pp = trafo(pp);
+	  s2->Project (pp);
+          for (PointIndex pj : Range(mesh.Points()))
+	    if (Dist2(mesh[pj], pp) < eps)
+              mesh.GetIdentifications().Add (pi, pj, nr);
+	}
+    }
 
   mesh.GetIdentifications().SetType(nr,Identifications::PERIODIC);
 }
@@ -392,15 +408,15 @@ void PeriodicIdentification :: IdentifyFaces (class Mesh & mesh)
 
 		    if (side == 1)
 		      {
-			if (mesh.GetIdentifications().Get (seg1[0], seg2[0]) &&
-			    mesh.GetIdentifications().Get (seg1[1], seg2[1]))
+			if (mesh.GetIdentifications().Used (seg1[0], seg2[0]) &&
+			    mesh.GetIdentifications().Used (seg1[1], seg2[1]))
 			  {
 			    foundother = 1;
 			    break;
 			  }
 			
-			if (mesh.GetIdentifications().Get (seg1[0], seg2[1]) &&
-			    mesh.GetIdentifications().Get (seg1[1], seg2[0]))
+			if (mesh.GetIdentifications().Used (seg1[0], seg2[1]) &&
+			    mesh.GetIdentifications().Used (seg1[1], seg2[0]))
 			  {
 			    foundother = 1;
 			    break;
@@ -408,15 +424,15 @@ void PeriodicIdentification :: IdentifyFaces (class Mesh & mesh)
 		      }
 		    else
 		      {
-			if (mesh.GetIdentifications().Get (seg2[0], seg1[0]) &&
-			    mesh.GetIdentifications().Get (seg2[1], seg1[1]))
+			if (mesh.GetIdentifications().Used (seg2[0], seg1[0]) &&
+			    mesh.GetIdentifications().Used (seg2[1], seg1[1]))
 			  {
 			    foundother = 1;
 			    break;
 			  }
 			
-			if (mesh.GetIdentifications().Get (seg2[0], seg1[1]) &&
-			    mesh.GetIdentifications().Get (seg2[1], seg1[0]))
+			if (mesh.GetIdentifications().Used (seg2[0], seg1[1]) &&
+			    mesh.GetIdentifications().Used (seg2[1], seg1[0]))
 			  {
 			    foundother = 1;
 			    break;
@@ -668,7 +684,7 @@ void CloseSurfaceIdentification :: IdentifySpecialPoints
 */
 
 int CloseSurfaceIdentification :: 
-Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 	      const TABLE<int> & specpoint2solid,
 	      const TABLE<int> & specpoint2surface) const
 {
@@ -826,7 +842,7 @@ Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 }
 
 int CloseSurfaceIdentification :: 
-Identifyable (const Point<3> & p1, const Point<3> & p2) const
+Identifiable (const Point<3> & p1, const Point<3> & p2) const
 {  
 //   if (domain)
 //     if (!domain->GetSolid()->IsIn (p1) || !domain->GetSolid()->IsIn (p2))
@@ -838,7 +854,7 @@ Identifyable (const Point<3> & p1, const Point<3> & p2) const
 
 
 int CloseSurfaceIdentification :: 
-IdentifyableCandidate (const SpecialPoint & sp1) const
+IdentifiableCandidate (const SpecialPoint & sp1) const
 {
   if (domain)
     if (!domain->GetSolid()->IsIn (sp1.p))
@@ -881,17 +897,21 @@ ShortEdge (const SpecialPoint & sp1, const SpecialPoint & sp2) const
 
 
 
-int CloseSurfaceIdentification :: 
-GetIdentifiedPoint (class Mesh & mesh,  int pi)
+PointIndex CloseSurfaceIdentification :: 
+GetIdentifiedPoint (class Mesh & mesh, PointIndex pi)
 {
   const Surface *snew;
   const Point<3> & p = mesh.Point (pi);
 
-  NgArray<int,PointIndex::BASE> identmap(mesh.GetNP());
+  idmap_type identmap(mesh.GetNP());
   mesh.GetIdentifications().GetMap (nr, identmap);
+  /*
   if (identmap.Get(pi))
     return identmap.Get(pi);
-
+  */
+  if (identmap[pi].IsValid())
+    return identmap[pi];
+  
   
   if (s1->PointOnSurface (p))
     snew = s2;
@@ -1164,15 +1184,15 @@ void CloseSurfaceIdentification :: IdentifyFaces (class Mesh & mesh)
 		      
 		      if (side == 1)
 			{
-			  if (mesh.GetIdentifications().Get (seg1[0], seg2[0]) &&
-			      mesh.GetIdentifications().Get (seg1[1], seg2[1]))
+			  if (mesh.GetIdentifications().Used (seg1[0], seg2[0]) &&
+			      mesh.GetIdentifications().Used (seg1[1], seg2[1]))
 			    {
 			      foundother = 1;
 			      break;
 			    }
 			  
-			  if (mesh.GetIdentifications().Get (seg1[0], seg2[1]) &&
-			      mesh.GetIdentifications().Get (seg1[1], seg2[0]))
+			  if (mesh.GetIdentifications().Used (seg1[0], seg2[1]) &&
+			      mesh.GetIdentifications().Used (seg1[1], seg2[0]))
 			    {
 			      foundother = 1;
 			      break;
@@ -1180,15 +1200,15 @@ void CloseSurfaceIdentification :: IdentifyFaces (class Mesh & mesh)
 			}
 		      else
 			{
-			  if (mesh.GetIdentifications().Get (seg2[0], seg1[0]) &&
-			      mesh.GetIdentifications().Get (seg2[1], seg1[1]))
+			  if (mesh.GetIdentifications().Used (seg2[0], seg1[0]) &&
+			      mesh.GetIdentifications().Used (seg2[1], seg1[1]))
 			    {
 			      foundother = 1;
 			      break;
 			    }
 			  
-			  if (mesh.GetIdentifications().Get (seg2[0], seg1[1]) &&
-			      mesh.GetIdentifications().Get (seg2[1], seg1[0]))
+			  if (mesh.GetIdentifications().Used (seg2[0], seg1[1]) &&
+			      mesh.GetIdentifications().Used (seg2[1], seg1[0]))
 			    {
 			      foundother = 1;
 			      break;
@@ -1225,7 +1245,7 @@ BuildSurfaceElements (NgArray<Segment> & segs,
   bool found = 0;
   int cntquads = 0;
 
-  NgArray<int,PointIndex::BASE>  identmap;
+  idmap_type identmap;
   identmap = 0;
 
   mesh.GetIdentifications().GetMap (nr, identmap);
@@ -1540,7 +1560,7 @@ void CloseEdgesIdentification :: IdentifySpecialPoints
 */
 
 int CloseEdgesIdentification :: 
-Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 	      const TABLE<int> & specpoint2solid,
 	      const TABLE<int> & specpoint2surface) const
 {
@@ -1646,8 +1666,8 @@ BuildSurfaceElements (NgArray<Segment> & segs,
       {
 	const Segment & s1 = segs.Get(i1);
 	const Segment & s2 = segs.Get(i2);
-	if (mesh.GetIdentifications().Get (s1[0], s2[1]) &&
-	    mesh.GetIdentifications().Get (s1[1], s2[0]))
+	if (mesh.GetIdentifications().Used (s1[0], s2[1]) &&
+	    mesh.GetIdentifications().Used (s1[1], s2[0]))
 	  {
 	    Element2d el(QUAD);
 	    el.PNum(1) = s1[0];

libsrc/csg/identify.hpp

@@ -38,13 +38,13 @@ namespace netgen
 
     /// can identify both special points (fixed direction)
     /// (identified points, same tangent)
-    virtual int Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+    virtual int Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 			      const TABLE<int> & specpoint2solid,			  
 			      const TABLE<int> & specpoint2surface) const;
     ///
-    virtual int Identifyable (const Point<3> & p1, const Point<3> & sp2) const;
+    virtual int Identifiable (const Point<3> & p1, const Point<3> & sp2) const;
     /// is it possible to identify sp1 with some other ?
-    virtual int IdentifyableCandidate (const SpecialPoint & sp1) const;
+    virtual int IdentifiableCandidate (const SpecialPoint & sp1) const;
   
     /// are points (if connected) by a short edge (direction anyhow) ?
     virtual int ShortEdge (const SpecialPoint & sp1, const SpecialPoint & sp2) const;
@@ -56,7 +56,7 @@ namespace netgen
     virtual void IdentifyFaces (class Mesh & mesh);
 
     /// get point on other surface, add entry in mesh identifications
-    virtual int GetIdentifiedPoint (class Mesh & mesh, int pi1);
+    virtual PointIndex GetIdentifiedPoint (class Mesh & mesh, PointIndex pi1);
 
     /// copy surfaces, or fill rectangles
     virtual void BuildSurfaceElements (NgArray<class Segment> & segs,
@@ -92,12 +92,12 @@ namespace netgen
 
 
     //  virtual void IdentifySpecialPoints (NgArray<class SpecialPoint> & points);
-    virtual int Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+    virtual int Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 			      const TABLE<int> & specpoint2solid,
 			      const TABLE<int> & specpoint2surface) const override;
 
-    virtual int Identifyable (const Point<3> & p1, const Point<3> & sp2) const override;
-    virtual int GetIdentifiedPoint (class Mesh & mesh, int pi1) override;
+    virtual int Identifiable (const Point<3> & p1, const Point<3> & sp2) const override;
+    virtual PointIndex GetIdentifiedPoint (class Mesh & mesh, PointIndex pi1) override;
     virtual void IdentifyPoints (class Mesh & mesh) override;
     virtual void IdentifyFaces (class Mesh & mesh) override;
     virtual void BuildSurfaceElements (NgArray<class Segment> & segs,
@@ -147,13 +147,13 @@ namespace netgen
 
 
     //  virtual void IdentifySpecialPoints (NgArray<class SpecialPoint> & points);
-    virtual int Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+    virtual int Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 			      const TABLE<int> & specpoint2solid,
 			      const TABLE<int> & specpoint2surface) const;
-    virtual int Identifyable (const Point<3> & p1, const Point<3> & sp2) const;
-    virtual int IdentifyableCandidate (const SpecialPoint & sp1) const;
+    virtual int Identifiable (const Point<3> & p1, const Point<3> & sp2) const;
+    virtual int IdentifiableCandidate (const SpecialPoint & sp1) const;
     virtual int ShortEdge (const SpecialPoint & sp1, const SpecialPoint & sp2) const;
-    virtual int GetIdentifiedPoint (class Mesh & mesh, int pi1);
+    virtual PointIndex GetIdentifiedPoint (class Mesh & mesh, PointIndex pi1);
     const Array<double> & GetSlices () const { return slices; }
     virtual void IdentifyPoints (class Mesh & mesh);
     virtual void IdentifyFaces (class Mesh & mesh);
@@ -197,7 +197,7 @@ namespace netgen
     virtual void GetData (ostream & ost) const;
 
     //  virtual void IdentifySpecialPoints (NgArray<class SpecialPoint> & points);
-    virtual int Identifyable (const SpecialPoint & sp1, const SpecialPoint & sp2,
+    virtual int Identifiable (const SpecialPoint & sp1, const SpecialPoint & sp2,
 			      const TABLE<int> & specpoint2solid,
 			      const TABLE<int> & specpoint2surface) const;
 

libsrc/csg/polyhedra.hpp

@@ -48,46 +48,63 @@ namespace netgen
 
   public:
     Polyhedra ();
-    virtual ~Polyhedra ();
+    virtual ~Polyhedra () override;
     static Primitive * CreateDefault ();
 
-    virtual INSOLID_TYPE BoxInSolid (const BoxSphere<3> & box) const;
+    virtual INSOLID_TYPE BoxInSolid (const BoxSphere<3> & box) const override;
     virtual INSOLID_TYPE PointInSolid (const Point<3> & p,
-				       double eps) const;
-    virtual INSOLID_TYPE VecInSolid (const Point<3> & p,
+				       double eps) const override;
+    virtual INSOLID_TYPE VecInSolidNew (const Point<3> & p,
+                                        const Vec<3> & v,
+                                        double eps, bool printing = false) const;
+    virtual INSOLID_TYPE VecInSolidOld (const Point<3> & p,
 				     const Vec<3> & v,
 				     double eps) const;
+    
+    virtual INSOLID_TYPE VecInSolid (const Point<3> & p,
+				     const Vec<3> & v,
+				     double eps) const override;
 
-    // checks if lim s->0 lim t->0  p + t(v1 + s v2) in solid
     virtual INSOLID_TYPE VecInSolid2 (const Point<3> & p,
 				      const Vec<3> & v1,
 				      const Vec<3> & v2,
-				      double eps) const;
+				      double eps) const override;
+    
+    virtual INSOLID_TYPE VecInSolid3 (const Point<3> & p,
+				      const Vec<3> & v1,
+				      const Vec<3> & v2,
+				      double eps) const override;
 
+    virtual INSOLID_TYPE VecInSolid4 (const Point<3> & p,
+				      const Vec<3> & v,
+				      const Vec<3> & v2,
+				      const Vec<3> & m,
+				      double eps) const override;
+    
     virtual void GetTangentialSurfaceIndices (const Point<3> & p, 
-					      NgArray<int> & surfind, double eps) const;
+					      NgArray<int> & surfind, double eps) const override;
 
 
     virtual void GetTangentialVecSurfaceIndices2 (const Point<3> & p, const Vec<3> & v1, const Vec<3> & v2,
-						  NgArray<int> & surfind, double eps) const;
+						  NgArray<int> & surfind, double eps) const override;
 
-    virtual void CalcSpecialPoints (NgArray<Point<3> > & pts) const;
+    virtual void CalcSpecialPoints (NgArray<Point<3> > & pts) const override;
     virtual void AnalyzeSpecialPoint (const Point<3> & pt, 
-				      NgArray<Point<3> > & specpts) const;
-    virtual Vec<3> SpecialPointTangentialVector (const Point<3> & p, int s1, int s2) const;
+				      NgArray<Point<3> > & specpts) const override;
+    virtual Vec<3> SpecialPointTangentialVector (const Point<3> & p, int s1, int s2) const override;
 
-    virtual int GetNSurfaces() const 
+    virtual int GetNSurfaces() const override
     { return planes.Size(); }
-    virtual Surface & GetSurface (int i) 
+    virtual Surface & GetSurface (int i) override
     { return *planes[i]; }
-    virtual const Surface & GetSurface (int i) const
+    virtual const Surface & GetSurface (int i) const override
     { return *planes[i]; }
 
-    virtual void GetPrimitiveData (const char *& classname, NgArray<double> & coeffs) const;
-    virtual void SetPrimitiveData (NgArray<double> & coeffs);
+    virtual void GetPrimitiveData (const char *& classname, NgArray<double> & coeffs) const override;
+    virtual void SetPrimitiveData (NgArray<double> & coeffs) override;
 
-    virtual void Reduce (const BoxSphere<3> & box);
-    virtual void UnReduce ();
+    virtual void Reduce (const BoxSphere<3> & box) override;
+    virtual void UnReduce () override;
 
     int AddPoint (const Point<3> & p);
     int AddFace (int pi1, int pi2, int pi3, int inputnum);

libsrc/csg/python_csg.cpp

@@ -1,12 +1,14 @@
 #ifdef NG_PYTHON
 
-#include <../general/ngpython.hpp>
-#include <core/python_ngcore.hpp>
-#include <csg.hpp>
+#include "../general/ngpython.hpp"
+#include "../core/python_ngcore.hpp"
+#include "csg.hpp"
 #include "../meshing/python_mesh.hpp"
+#include "../general/gzstream.h"
 
 
 using namespace netgen;
+using namespace pybind11::literals;
 
 namespace netgen
 {
@@ -170,7 +172,8 @@ namespace netgen
 
 DLL_HEADER void ExportCSG(py::module &m)
 {
-  py::class_<SplineGeometry<2>> (m, "SplineCurve2d")
+  py::class_<SplineGeometry<2>, shared_ptr<SplineGeometry<2>>>
+    (m, "SplineCurve2d")
     .def(py::init<>())
     .def ("AddPoint", FunctionPointer
           ([] (SplineGeometry<2> & self, double x, double y)
@@ -178,16 +181,16 @@ DLL_HEADER void ExportCSG(py::module &m)
              self.geompoints.Append (GeomPoint<2> (Point<2> (x,y)));
              return self.geompoints.Size()-1;
            }))
-    .def ("AddSegment", FunctionPointer
-          ([] (SplineGeometry<2> & self, int i1, int i2)
-           {
-             self.splines.Append (new LineSeg<2> (self.geompoints[i1], self.geompoints[i2]));
-           }))
-    .def ("AddSegment", FunctionPointer
-          ([] (SplineGeometry<2> & self, int i1, int i2, int i3)
-           {
-             self.splines.Append (new SplineSeg3<2> (self.geompoints[i1], self.geompoints[i2], self.geompoints[i3]));
-           }))
+    .def ("AddSegment", [] (SplineGeometry<2> & self, int i1, int i2,
+                            string bcname, double maxh)
+    {
+      self.splines.Append (new LineSeg<2> (self.geompoints[i1], self.geompoints[i2], maxh, bcname));
+    }, "p1"_a, "p2"_a, "bcname"_a="default", "maxh"_a=1e99)
+    .def ("AddSegment", [] (SplineGeometry<2> & self, int i1, int i2,
+                            int i3, string bcname, double maxh)
+    {
+      self.splines.Append (new SplineSeg3<2> (self.geompoints[i1], self.geompoints[i2], self.geompoints[i3], bcname, maxh));
+    }, "p1"_a, "p2"_a, "p3"_a, "bcname"_a="default", "maxh"_a=1e99)
     ;
 
   py::class_<SplineGeometry<3>,shared_ptr<SplineGeometry<3>>> (m,"SplineCurve3d")
@@ -322,21 +325,38 @@ DLL_HEADER void ExportCSG(py::module &m)
                                            Solid * sol = new Solid (torus);
                                            return make_shared<SPSolid> (sol);
                                          }));
-  m.def ("Revolution", FunctionPointer([](Point<3> p1, Point<3> p2,
-                                            const SplineGeometry<2> & spline)
-                                         {
-                                           Revolution * rev = new Revolution (p1, p2, spline);
-                                           Solid * sol = new Solid(rev);
-                                           return make_shared<SPSolid> (sol);
-                                         }));
-  m.def ("Extrusion", FunctionPointer([](const SplineGeometry<3> & path,
-					 const SplineGeometry<2> & profile,
-					 Vec<3> n)
-                                         {
-                                           Extrusion * extr = new Extrusion (path,profile,n);
-                                           Solid * sol = new Solid(extr);
-                                           return make_shared<SPSolid> (sol);
-                                         }));
+  m.def ("Revolution", [](Point<3> p1, Point<3> p2,
+                          shared_ptr<SplineGeometry<2>> spline)
+  {
+    Revolution * rev = new Revolution (p1, p2, spline);
+    Solid * sol = new Solid(rev);
+    return make_shared<SPSolid> (sol);
+  });
+  m.def ("Extrusion", [](shared_ptr<SplineGeometry<3>> path,
+                         shared_ptr<SplineGeometry<2>> profile,
+                         Vec<3> d)
+  {
+    Extrusion * extr = new Extrusion (path,profile,d);
+    Solid * sol = new Solid(extr);
+    return make_shared<SPSolid> (sol);
+  }, py::arg("path"), py::arg("profile"), py::arg("d"),
+     R"delimiter(A body of extrusion is defined by its profile
+(which has to be a closed, clockwiseoriented 2D curve),
+ by a path (a 3D curve) and a vector d. It is constructed
+ as follows: Take a point p on the path and denote the
+ (unit-)tangent of the path in this point by t. If we cut
+ the body by the plane given by p and t as normal vector,
+ the cut is the profile. The profile is oriented by the
+ (local) y-direction `y:=d−(d·t)t` and the (local) x-direction
+ `x:=t \times y`.
+The following points have to be noticed:
+ * If the path is not closed, then also the body is NOT closed.
+   In this case e.g. planes or orthobricks have to be used to
+   construct a closed body.
+ * The path has to be smooth, i.e. the tangents at the end- resp.
+   start-point of two consecutive spline or line patches have to
+   have the same directions.
+)delimiter");
   m.def("EllipticCone", [](const Point<3>& a, const Vec<3>& v, const Vec<3>& w,
                             double h, double r)
         {
@@ -354,6 +374,35 @@ When r =1, the truncated elliptic cone becomes an elliptic cylinder.
 When r tends to zero, the truncated elliptic cone tends to a full elliptic cone.
 However, when r = 0, the top part becomes a point(tip) and meshing fails!
 )raw_string");
+
+  m.def("Polyhedron", [](py::list points, py::list faces)
+  {
+    auto poly = new Polyhedra();
+    for(auto p : points)
+      poly->AddPoint(py::cast<Point<3>>(p));
+    int fnr = 0;
+    for(auto face : faces)
+      {
+        auto lface = py::cast<py::list>(face);
+        if(py::len(lface) == 3)
+          poly->AddFace(py::cast<int>(lface[0]),
+                        py::cast<int>(lface[1]),
+                        py::cast<int>(lface[2]),
+                        fnr++);
+        else if(py::len(lface) == 4)
+          {
+            poly->AddFace(py::cast<int>(lface[0]),
+                          py::cast<int>(lface[1]),
+                          py::cast<int>(lface[2]),
+                          fnr);
+            poly->AddFace(py::cast<int>(lface[0]),
+                          py::cast<int>(lface[2]),
+                          py::cast<int>(lface[3]),
+                          fnr++);
+          }
+      }
+    return make_shared<SPSolid>(new Solid(poly));
+  });
   
   m.def ("Or", FunctionPointer([](shared_ptr<SPSolid> s1, shared_ptr<SPSolid> s2)
                                  {
@@ -534,8 +583,8 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
             NgArray<int> si1, si2;
             s1->GetSolid()->GetSurfaceIndices (si1);
             s2->GetSolid()->GetSurfaceIndices (si2);
-            cout << "surface ids1 = " << si1 << endl;
-            cout << "surface ids2 = " << si2 << endl;
+            cout << IM(3) << "surface ids1 = " << si1 << endl;
+            cout << IM(3) << "surface ids2 = " << si2 << endl;
 
             Flags flags;
             const TopLevelObject * domain = nullptr;
@@ -627,7 +676,7 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
     .def("Draw", FunctionPointer
          ([] (shared_ptr<CSGeometry> self)
           {
-             self->FindIdenticSurfaces(1e-6);
+             self->FindIdenticSurfaces(1e-8 * self->MaxSize());
              self->CalcTriangleApproximation(0.01, 20);
              ng_geometry = self;
           })
@@ -657,8 +706,8 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
                auto surf = csg_geo->GetSurface(i);
                surfnames.push_back(surf->GetBCName());
              }
-           csg_geo->FindIdenticSurfaces(1e-6);
-           csg_geo->CalcTriangleApproximation(0.01,100);
+           csg_geo->FindIdenticSurfaces(1e-8 * csg_geo->MaxSize());
+           csg_geo->CalcTriangleApproximation(0.01,20);
            auto nto = csg_geo->GetNTopLevelObjects();
            size_t np = 0;
            size_t ntrig = 0;
@@ -707,10 +756,8 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
            {
              MeshingParameters mp;
              if(pars) mp = *pars;
-             {
-               py::gil_scoped_acquire aq;
-               CreateMPfromKwargs(mp, kwargs);
-             }
+             CreateMPfromKwargs(mp, kwargs);
+             py::gil_scoped_release gil_rel;
              auto mesh = make_shared<Mesh>();
              SetGlobalMesh (mesh);
              mesh->SetGeometry(geo);
@@ -721,8 +768,7 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
                throw Exception("Meshing failed!");
              return mesh;
            }, py::arg("mp") = nullptr,
-       meshingparameter_description.c_str(),
-    py::call_guard<py::gil_scoped_release>())
+       meshingparameter_description.c_str())
     ;
 
   m.def("Save", FunctionPointer 

libsrc/csg/revolution.cpp

@@ -1,4 +1,5 @@
 #include <mystdlib.h>
+#include <core/register_archive.hpp>
 
 #include <linalg.hpp>
 #include <csg.hpp>
@@ -48,6 +49,8 @@ namespace netgen
     isfirst(first), islast(last), spline(&spline_in), p0(p), v_axis(vec),  id(id_in)
   {    
     deletable = false;
+    maxh = spline_in.GetMaxh();
+    bcname = spline_in.GetBCName();
     Init();
   }
 
@@ -640,10 +643,10 @@ namespace netgen
 
   Revolution :: Revolution(const Point<3> & p0_in,
 			   const Point<3> & p1_in,
-			   const SplineGeometry<2> & spline_in) :
-    p0(p0_in), p1(p1_in)
+			   shared_ptr<SplineGeometry<2>> spline_in) :
+    p0(p0_in), p1(p1_in), splinegeo(spline_in)
   {
-    auto nsplines = spline_in.GetNSplines();
+    auto nsplines = spline_in->GetNSplines();
     surfaceactive.SetSize(0);
     surfaceids.SetSize(0);
 
@@ -651,25 +654,42 @@ namespace netgen
 
     v_axis.Normalize();
 
-    if(spline_in.GetSpline(0).StartPI()(1) <= 0. &&
-       spline_in.GetSpline(nsplines-1).EndPI()(1) <= 0.)
+    if(spline_in->GetSpline(0).StartPI()(1) <= 0. &&
+       spline_in->GetSpline(nsplines-1).EndPI()(1) <= 0.)
       type = 2;
-    else if (Dist(spline_in.GetSpline(0).StartPI(),
-		  spline_in.GetSpline(nsplines-1).EndPI()) < 1e-7)
+    else if (Dist(spline_in->GetSpline(0).StartPI(),
+		  spline_in->GetSpline(nsplines-1).EndPI()) < 1e-7)
       type = 1;
     else
       cerr << "Surface of revolution cannot be constructed" << endl;
 
-    for(int i=0; i<spline_in.GetNSplines(); i++)
+    for(int i=0; i<spline_in->GetNSplines(); i++)
       {
-	RevolutionFace * face = new RevolutionFace(spline_in.GetSpline(i),
-						   p0,v_axis,
-						   type==2 && i==0,
-						   type==2 && i==spline_in.GetNSplines()-1);
-	faces.Append(face);
-	surfaceactive.Append(1);
+	faces.Append(new RevolutionFace
+                     (spline_in->GetSpline(i),
+                      p0,v_axis,
+                      type==2 && i==0,
+                      type==2 && i==spline_in->GetNSplines()-1));
+        surfaceactive.Append(1);
 	surfaceids.Append(0);
       }
+
+    // checking
+    if (type == 2)
+      {
+        auto t0 = spline_in->GetSpline(0).GetTangent(0);
+        cout << "tstart (must be vertically): " << t0 << endl;
+
+        auto tn = spline_in->GetSpline(nsplines-1).GetTangent(1);
+        cout << "tend (must be vertically): " << tn << endl;
+
+        for (int i = 0; i < nsplines-1; i++)
+          {
+            auto ta = spline_in->GetSpline(i).GetTangent(1);
+            auto tb = spline_in->GetSpline(i+1).GetTangent(0);
+            cout << "sin (must not be 0) = " << abs(ta(0)*tb(1)-ta(1)*tb(0)) / (Abs(ta)*Abs(tb)); 
+          }
+      }
   }
   
   Revolution::~Revolution()
@@ -761,11 +781,12 @@ namespace netgen
     Point<2> p2d;
     faces[0]->CalcProj(p,p2d);
 
-    int intersections_before(0), intersections_after(0);
+    [[maybe_unused]] int intersections_before(0), intersections_after(0);
     double randomx = 7.42357;
     double randomy = 1.814756;
-    randomx *= 1./sqrt(randomx*randomx+randomy*randomy);
-    randomy *= 1./sqrt(randomx*randomx+randomy*randomy);
+    double randomlen = sqrt(randomx*randomx+randomy*randomy);
+    randomx *= 1./randomlen;
+    randomy *= 1./randomlen;
     
 
     const double a = randomy;
@@ -930,6 +951,67 @@ namespace netgen
 
     return VecInSolid(p,v1+0.01*v2,eps);
   }
+
+  void Revolution ::
+  GetTangentialVecSurfaceIndices2 (const Point<3> & p, const Vec<3> & v1, const Vec<3> & v2,
+                                   NgArray<int> & surfind, double eps) const
+  {
+    *testout << "tangentialvecsurfind2, p = " << p << endl;
+    for (int i = 0; i < faces.Size(); i++)
+      if (faces[i]->PointInFace (p, eps))
+        {
+          *testout << "check face " << i << endl;
+          Point<2> p2d;
+          Vec<2> v12d;
+          faces[i]->CalcProj(p,p2d,v1,v12d);
+          *testout << "v12d = " << v12d << endl;
+          auto & spline = faces[i]->GetSpline();
+          if (Dist2 (spline.StartPI(), p2d) < sqr(eps))
+            {
+              *testout << "start pi" << endl;
+              Vec<2> tang = spline.GetTangent(0);
+              double ip = tang*v12d;
+              *testout << "ip = " << ip << endl;
+              if (ip > eps)
+                surfind.Append(GetSurfaceId(i));
+              else if (ip > -eps)
+                {
+                  Vec<2> v22d;
+                  faces[i]->CalcProj(p,p2d,v2,v22d);
+                  double ip2 = tang*v22d;
+                  *testout << "ip2 = " << ip2 << endl;
+                  if (ip2 > -eps)
+                    surfind.Append(GetSurfaceId(i));                    
+                }
+            }
+          else if (Dist2 (faces[i]->GetSpline().EndPI(), p2d) < sqr(eps))
+            {
+              *testout << "end pi" << endl;
+              
+              Vec<2> tang = spline.GetTangent(1);
+              double ip = tang*v12d;
+              *testout << "ip = " << ip << endl;
+              if (ip < -eps)
+                surfind.Append(GetSurfaceId(i));
+              else if (ip < eps)
+                {
+                  Vec<2> v22d;
+                  faces[i]->CalcProj(p,p2d,v2,v22d);
+                  double ip2 = tang*v22d;
+                  *testout << "ip2 = " << ip2 << endl;                  
+                  if (ip2 < eps)
+                    surfind.Append(GetSurfaceId(i));                    
+                }
+            }
+          else
+            {
+              *testout << "inner point" << endl;
+              surfind.Append(GetSurfaceId(i));
+            }
+        }
+  }
+
+
   
   int Revolution :: GetNSurfaces() const
   {

libsrc/csg/revolution.hpp

@@ -67,7 +67,10 @@ namespace netgen
     virtual double MaxCurvature () const;
     //virtual double MaxCurvatureLoc (const Point<3> & /* c */ , 
     //				  double /* rad */) const;
-
+    
+    Point<3> P0() const { return p0; }
+    Vec<3> Axis() const { return v_axis; }
+    
     virtual void Project (Point<3> & p) const;
 
     virtual Point<3> GetSurfacePoint () const;
@@ -111,14 +114,15 @@ namespace netgen
     int type;
   
 
-    NgArray<RevolutionFace*> faces;
+    Array<RevolutionFace*> faces;
+    shared_ptr<SplineGeometry<2>> splinegeo;
 
     mutable int intersecting_face;
 
   public:
     Revolution(const Point<3> & p0_in,
 	       const Point<3> & p1_in,
-	       const SplineGeometry<2> & spline_in);
+	       shared_ptr<SplineGeometry<2>> spline_in);
     // default constructor for archive
     Revolution() {}
 
@@ -155,7 +159,10 @@ namespace netgen
 				      const Vec<3> & v2,
 				      double eps) const;
 
-  
+    virtual void GetTangentialVecSurfaceIndices2 (const Point<3> & p, const Vec<3> & v1, const Vec<3> & v2,
+						  NgArray<int> & surfind, double eps) const;
+
+    
     virtual int GetNSurfaces() const;
     virtual Surface & GetSurface (int i = 0);
     virtual const Surface & GetSurface (int i = 0) const;

libsrc/csg/singularref.cpp

@@ -53,7 +53,7 @@ void SingularEdge :: FindPointsOnEdge (class Mesh & mesh)
 
   for (SegmentIndex si = 0; si < mesh.GetNSeg(); si++)
     {
-      INDEX_2 i2 (mesh[si][0], mesh[si][1]);
+      PointIndices<2> i2 (mesh[si][0], mesh[si][1]);
       /*
       
       bool onedge = 1;
@@ -93,8 +93,8 @@ void SingularEdge :: FindPointsOnEdge (class Mesh & mesh)
 	{
 	  segms.Append (i2);
 	  //	  PrintMessage (5, "sing segment ", i2.I1(), " - ", i2.I2());
-	  points.Append (mesh[ PointIndex (i2.I1())]);
-	  points.Append (mesh[ PointIndex (i2.I2())]);
+	  points.Append (mesh[i2.I1()]);
+	  points.Append (mesh[i2.I2()]);
 	  mesh[si].singedge_left = factor;
 	  mesh[si].singedge_right = factor;
 	}	    
@@ -153,8 +153,8 @@ void SingularPoint :: FindPoints (class Mesh & mesh)
   NgArray<int> surfk, surf;
 
 
-  for (PointIndex pi = PointIndex::BASE; 
-       pi < mesh.GetNP()+PointIndex::BASE; pi++)
+  for (PointIndex pi = IndexBASE<PointIndex>(); 
+       pi < mesh.GetNP()+IndexBASE<PointIndex>(); pi++)
     {
       if (mesh[pi].Type() != FIXEDPOINT) continue;
       const Point<3> p = mesh[pi];
@@ -168,7 +168,7 @@ void SingularPoint :: FindPoints (class Mesh & mesh)
 	  for (int k = 1; k <= 3; k++)
 	    {
 	      const Solid * solk(NULL);
-	      Solid *tansol;
+	      // Solid *tansol;
 	      switch (k)
 		{
 		case 1:  solk = sol1; break;
@@ -176,7 +176,7 @@ void SingularPoint :: FindPoints (class Mesh & mesh)
 		case 3:  solk = sol3; break;
 		}
 
-	      solk -> TangentialSolid (p, tansol, surfk, 1e-3);
+	      auto tansol = solk -> TangentialSolid (p, surfk, 1e-3);
 	      (*testout) << "Tansol = " << *tansol << endl;
 
 	      if (!tansol) continue;
@@ -195,7 +195,7 @@ void SingularPoint :: FindPoints (class Mesh & mesh)
 		if (!surf.Contains (surfk[i]))
 		  surf.Append (surfk[i]);
 	      
-	      delete tansol;
+	      // delete tansol;
 	    }
 
 	  if (surf.Size() < 3) continue;