Mod: core: improve pipeline

Fix: openfoam: wrong path expressions

anisotropy/cli/cli.py

@@ -65,7 +65,7 @@ def init(path, verbose):
 @click.option(
     "-f", "--force", "overwrite",
     is_flag = True,
-    # default = False,
+    default = None,
     help = "Overwrite existing entries"
 )
 @click.option(
@@ -135,6 +135,7 @@ def compute(path, configFile, nprocs, stage, overwrite, params, verbose, executi
         if v is not None:
             config.update(**{ k: v })
 
+    print(config.options)
     if pid:
         pid = pathlib.Path(pid).resolve()
 

anisotropy/core/runner.py

@@ -12,6 +12,7 @@ from anisotropy.database import Database, tables
 from anisotropy import shaping
 from anisotropy import meshing
 from anisotropy import solving
+from anisotropy import openfoam
 
 from . import config as core_config
 from . import postprocess
@@ -51,6 +52,24 @@ class UltimateRunner(object):
         # Parameters
         self.queue = []
 
+    def _query_database(self, tableName: str, to_dict: bool = False):
+        tableName = tableName.lower()
+        get_row = {
+            "shape": self.database.getShape,
+            "mesh": self.database.getMesh,
+            "flowonephase": self.database.getFlowOnephase
+        }[tableName]
+        #   query
+        args = (
+            self.config.params["label"],
+            self.config.params["direction"],
+            self.config.params["alpha"],
+            self.exec_id,
+            to_dict
+        )
+
+        return get_row(*args)
+
     def prepare_database(self):
         # create a row in each table for the current case
         #   shape
@@ -128,25 +147,16 @@ class UltimateRunner(object):
 
     def compute_shape(self):
         params = self.config.params
-        shapeParams = self.database.getShape(
+        shapeParams = self._query_database("shape")
-            params["label"],
-            params["direction"],
-            params["alpha"],
-            self.exec_id
-        )
 
         logger.info("Computing shape for {} with direction = {} and alpha = {}".format(
             params["label"], params["direction"], params["alpha"]
         ))
         timer = utils.Timer()
         
-        #   check
+        #   check physical existence
-        if (
+        if self.shapefile.exists() and self.config["overwrite"] is not True:
-            self.shapefile.exists() and 
+            logger.info("Shape exists, skipping ...")
-            shapeParams.shapeStatus == "done" and 
-            not self.config["overwrite"]
-        ):
-            logger.info("Shape exists. Skipping ...")
             return
         
         #
@@ -187,25 +197,16 @@ class UltimateRunner(object):
 
     def compute_mesh(self):
         params = self.config.params
-        meshParams = self.database.getMesh(
+        meshParams = self._query_database("mesh")
-            params["label"],
-            params["direction"],
-            params["alpha"],
-            self.exec_id
-        )
 
         logger.info("Computing mesh for {} with direction = {} and alpha = {}".format(
             params["label"], params["direction"], params["alpha"]
         ))
         timer = utils.Timer()
 
-        #   check 
+        #   check physical existence
-        if (
+        if self.meshfile.exists() and self.config["overwrite"] is not True:
-            self.meshfile.exists() and 
+            logger.info("Mesh exists, skipping ...")
-            meshParams.meshStatus == "done" and 
-            not self.config["overwrite"]
-        ):
-            logger.info("Mesh exists. Skipping ...")
             return
         
         elif not self.shapefile.exists():
@@ -246,21 +247,15 @@ class UltimateRunner(object):
 
     def compute_flow(self):
         params = self.config.params
-        query = (
+        flowParams = self._query_database("flowonephase")
-            params["label"],
-            params["direction"],
-            params["alpha"],
-            self.exec_id
-        )
-        flowParams = self.database.getFlowOnephase(*query)
 
         logger.info("Computing flow for {} with direction = {} and alpha = {}".format(
             params["label"], params["direction"], params["alpha"]
         ))
         timer = utils.Timer()
         
-        #   check
+        #   check physical existence
-        if flowParams.flowStatus == "done" and not self.config["overwrite"]:
+        if openfoam.FoamCase(path = self.casepath).is_converged() and not self.config["overwrite"]:
             logger.info("Solution exists. Skipping ...")
             return
         
@@ -277,7 +272,7 @@ class UltimateRunner(object):
         self.database.csave(flowParams)            
 
         #
-        flowParamsDict = self.database.getFlowOnephase(*query, to_dict = True)
+        flowParamsDict = self._query_database("flowonephase", to_dict = True)
 
         try:
             #   load shape
@@ -309,18 +304,14 @@ class UltimateRunner(object):
 
     def compute_postprocess(self):
         params = self.config.params
-        flowParams = self.database.getFlowOnephase(
+        flowParams = self._query_database("flowonephase")
-            params["label"],
-            params["direction"],
-            params["alpha"],
-            self.exec_id
-        )
 
         logger.info("Computing post process for {} with direction = {} and alpha = {}".format(
             params["label"], params["direction"], params["alpha"]
         ))
 
         if flowParams.flowStatus == "done":
+            if flowParams.flowRate is None and self.config["overwrite"] is not True:
                 flowParams.flowRate = postprocess.flowRate("outlet", self.casepath)
 
         self.database.csave(flowParams)
@@ -335,15 +326,43 @@ class UltimateRunner(object):
         
         for current in stages:
             if current == "shape":
+                params = self._query_database("shape")
+
+                #   check database entry
+                if params.shapeStatus == "done" and self.config["overwrite"] is not True:
+                    logger.info("Successful shape entry exists, skipping ...")
+                    continue
+
                 self.compute_shape()
 
             if current == "mesh":
+                params = self._query_database("mesh")
+
+                #   check database entry
+                if params.meshStatus == "done" and self.config["overwrite"] is not True:
+                    logger.info("Successful mesh entry exists, skipping ...")
+                    continue
+
                 self.compute_mesh()
 
             if current == "flow":
+                params = self._query_database("flowonephase")
+
+                #   check database entry
+                if params.flowStatus == "done" and self.config["overwrite"] is not True:
+                    logger.info("Successful flow entry exists, skipping ...")
+                    continue
+
                 self.compute_flow()
 
             if current == "postProcess":
+                params = self._query_database("flowonephase")
+
+                #   check database entry
+                # if params.flowStatus == "done" and self.config["overwrite"] is not True:
+                #    logger.info("Successful flow entry exists, skipping ...")
+                #    continue
+
                 self.compute_postprocess()
             
             if current == stage or current == "all":

anisotropy/database/database.py

@@ -57,7 +57,7 @@ class Database(pw.SqliteDatabase):
 
         return self
 
-    def csave(self, table: pw.Model, tries: int = 100):
+    def csave(self, table: pw.Model, tries: int = 5000):
         """Try to save data from model to the database ignoring
         peewee.OperationalError. Usefull for concurrent processes.
 
@@ -82,7 +82,12 @@ class Database(pw.SqliteDatabase):
                 self.close()
                 break
 
-    def getExecution(self, idn: int) -> tables.Execution | None:
+    def getExecution(
+        self, 
+        idn: int,
+        to_dict: bool = False,
+        **kwargs
+    ) -> tables.Execution | None:
         """Get execution entry from database.
 
         :param idn:
@@ -93,6 +98,10 @@ class Database(pw.SqliteDatabase):
         query = tables.Execution.select().where(tables.Execution.exec_id == idn)
 
         with self:
+            if to_dict:
+                table = query.dicts().get() if query.exists() else None
+            
+            else:
                 table = query.get() if query.exists() else None
 
         return table
@@ -116,6 +125,7 @@ class Database(pw.SqliteDatabase):
         direction: list[float] | ndarray = None, 
         alpha: float = None, 
         execution: int = None, 
+        to_dict: bool = False,
         **kwargs
     ) -> tables.Shape | None:
         """Get shape entry from database.
@@ -145,6 +155,10 @@ class Database(pw.SqliteDatabase):
         )
 
         with self:
+            if to_dict:
+                table = query.dicts().get() if query.exists() else None
+            
+            else:
                 table = query.get() if query.exists() else None
 
         return table
@@ -155,6 +169,7 @@ class Database(pw.SqliteDatabase):
         direction: list[float] | ndarray = None, 
         alpha: float = None, 
         execution: int = None, 
+        to_dict: bool = False,
         **kwargs
     ) -> tables.Mesh | None:
         """Get mesh entry from database.
@@ -185,6 +200,10 @@ class Database(pw.SqliteDatabase):
         )
 
         with self:
+            if to_dict:
+                table = query.dicts().get() if query.exists() else None
+            
+            else:
                 table = query.get() if query.exists() else None
 
         return table

anisotropy/gui/layouts/visualization.py

@@ -231,7 +231,7 @@ layout = html.Div([
     ]
 )
 def plotDraw(clicks, execution, structure, direction, data):
-    from peewee import JOIN
+    import peewee as pw
     from anisotropy.database import Database, tables
     import json
     from pandas import DataFrame
@@ -263,20 +263,32 @@ def plotDraw(clicks, execution, structure, direction, data):
     else:
         select = (tables.Shape.alpha, column)
 
-    query = (
+    """query = (
         tables.Shape
         .select(*select)
         .join(tables.Execution, JOIN.LEFT_OUTER)
         .switch(tables.Shape)
         .join(tables.Mesh, JOIN.LEFT_OUTER)
+        #.switch(tables.Shape)
+        .join(tables.FlowOnephase, JOIN.LEFT_OUTER)
         .switch(tables.Shape)
-        # .join(tables.FlowOnephase, JOIN.LEFT_OUTER)
-        # .switch(tables.Shape)
         .where(
             tables.Shape.exec_id == execution,
             tables.Shape.label == structure,
         )
+    )"""
+    query = model.select(*select)
+    idn = db.tables.index(model)
+
+    for table in reversed(db.tables[ :idn]):
+        query = query.join(table, pw.JOIN.LEFT_OUTER)
+    
+    query = query.switch(tables.Shape)
+    query = query.where(
+        tables.Shape.exec_id == execution,
+        tables.Shape.label == structure,
     )
+    query = query.order_by(tables.Shape.alpha)
     
     if not direction == "all":
         query = query.where(tables.Shape.direction == json.loads(direction))

anisotropy/openfoam/case.py

@@ -6,6 +6,8 @@ import os
 import shutil
 import re
 import pathlib
+import io
+from lz.reversal import reverse as lz_reverse
 
 from . import FoamFile
 
@@ -72,23 +74,23 @@ class FoamCase(object):
         path = pathlib.Path(path or self.path or "")
 
         for file in self._files:
-            path /= (
+            path_ = path / (
                 file.location + "/" + file.object 
                 if file.location else file.object
             )
 
-            file.read(path.resolve())
+            file.read(path_.resolve())
 
     def remove(self, path: str = None):
         path = pathlib.Path(path or self.path or "")
 
         for file in self._files:
-            path /= (
+            path_ = path / (
                 file.location + "/" + file.object 
                 if file.location else file.object
             )
 
-            file.remove(path.resolve())
+            file.remove(path_.resolve())
 
     def clean(self, included: list = ["0", "constant", "system"]):
         regxs = [
@@ -105,10 +107,16 @@ class FoamCase(object):
 
         for root, dirs, files in os.walk(os.path.abspath("")):
             for _dir in dirs:
-                excluded += [ os.path.join(root, _dir) for regx in regxs if re.match(regx, _dir) ]
+                excluded += [ 
+                    os.path.join(root, _dir) 
+                    for regx in regxs if re.match(regx, _dir) 
+                ]
 
             for file in files:
-                excluded += [ os.path.join(root, file) for regx in regxs if re.match(regx, file) ]
+                excluded += [ 
+                    os.path.join(root, file) 
+                    for regx in regxs if re.match(regx, file) 
+                ]
 
         for file in excluded:
             if os.path.split(file)[1] not in included and os.path.exists(file):
@@ -128,3 +136,39 @@ class FoamCase(object):
         path = pathlib.Path(self._backpath or "").resolve()
 
         os.chdir(path)
+
+    def is_converged(self, path: str = None) -> None | bool:
+        path = pathlib.Path(path or self.path or "").resolve()
+        controlDict = FoamFile()
+
+        if controlDict.exists(path / "system" / "controlDict"):
+            controlDict.read(path / "system" / "controlDict")
+        
+        else:
+            return None
+        
+        application = controlDict.get("application")
+
+        if application is None:
+            return None
+        
+        logfile = (
+            path / f"{ application }.log" 
+            if (path / f"{ application }.log").exists() else (path / f"log.{ application }")
+        )
+        status = False
+        
+        if logfile.exists():
+            with open(logfile, "r") as infile:
+                limit = 30
+
+                for line in lz_reverse(infile, batch_size = io.DEFAULT_BUFFER_SIZE):
+                    if not line.find("End") < 0:
+                        status = True
+                    
+                    if limit <= 0:
+                        status = False
+                    
+                    limit -= 1
+
+        return status

anisotropy/openfoam/conversion.py

@@ -22,6 +22,10 @@ def read_foamfile(filename: str) -> tuple[dict, dict]:
     header = ppf.header or {}
     content = ppf.content or {}
 
+    if type(content) == dict:
+        if content.get("internalField"):
+            content["internalField"] = np.asarray(content["internalField"])
+
     return header, content
 
 

anisotropy/openfoam/file.py

@@ -68,11 +68,17 @@ class FoamFile(object):
 
     @property
     def location(self) -> str:
-        return self.header.get("location").replace('"', "")
+        return (
+            self.header.get("location").replace('"', "") 
+            if self.header.get("location") else None
+        )
 
     def __getitem__(self, key):
         return self.content[key]
 
+    def get(self, key, default = None):
+        return self.content.get(key, default)
+
     def __setitem__(self, key, value):
         self.content[key] = value
 
@@ -99,6 +105,15 @@ class FoamFile(object):
 
         return FoamCase([ self, file ])
 
+    def exists(self, filename: str = None) -> bool:
+        filename = filename or (
+            self.location + "/" + self.object 
+            if self.location else self.object
+        )
+        path = pathlib.Path(filename).resolve()
+
+        return path.exists() and path.is_file()
+
     def read(self, filename: str = None):
         """Read a FoamFile.
 
@@ -108,8 +123,8 @@ class FoamFile(object):
         :return:
             Self.
         """
-        filename = (
+        filename = filename or (
-            filename or self.location + "/" + self.object 
+            self.location + "/" + self.object 
             if self.location else self.object
         )
         path = pathlib.Path(filename).resolve()
@@ -131,8 +146,8 @@ class FoamFile(object):
             Path to the file. If None, use location and object from header with
             the current working directory.
         """  
-        filename = (
+        filename = filename or (
-            filename or self.location + "/" + self.object 
+            self.location + "/" + self.object 
             if self.location else self.object
         )
         path = pathlib.Path(filename).resolve()
@@ -146,8 +161,8 @@ class FoamFile(object):
             Path to the file. If None, use location and object from header with
             the current working directory.
         """  
-        filename = (
+        filename = filename or (
-            filename or self.location + "/" + self.object 
+            self.location + "/" + self.object 
             if self.location else self.object
         )
         path = pathlib.Path(filename).resolve()