From fc1b8432cb8395b00256324ddbf0d0d8a16c2e12 Mon Sep 17 00:00:00 2001 From: L-Nafaryus Date: Fri, 18 Feb 2022 00:28:05 +0500 Subject: [PATCH] Mod: core: improve pipeline Fix: openfoam: wrong path expressions --- anisotropy/cli/cli.py | 5 +- anisotropy/core/runner.py | 107 ++++++++++++++---------- anisotropy/database/database.py | 29 +++++-- anisotropy/gui/layouts/visualization.py | 22 +++-- anisotropy/openfoam/case.py | 56 +++++++++++-- anisotropy/openfoam/conversion.py | 4 + anisotropy/openfoam/file.py | 29 +++++-- 7 files changed, 183 insertions(+), 69 deletions(-) diff --git a/anisotropy/cli/cli.py b/anisotropy/cli/cli.py index 076f464..b8dc73b 100644 --- a/anisotropy/cli/cli.py +++ b/anisotropy/cli/cli.py @@ -65,7 +65,7 @@ def init(path, verbose): @click.option( "-f", "--force", "overwrite", is_flag = True, - # default = False, + default = None, help = "Overwrite existing entries" ) @click.option( @@ -130,11 +130,12 @@ def compute(path, configFile, nprocs, stage, overwrite, params, verbose, executi "stage": stage, "overwrite": overwrite } - + for k, v in args.items(): if v is not None: config.update(**{ k: v }) + print(config.options) if pid: pid = pathlib.Path(pid).resolve() diff --git a/anisotropy/core/runner.py b/anisotropy/core/runner.py index f01afe8..d98e8ee 100644 --- a/anisotropy/core/runner.py +++ b/anisotropy/core/runner.py @@ -12,6 +12,7 @@ from anisotropy.database import Database, tables from anisotropy import shaping from anisotropy import meshing from anisotropy import solving +from anisotropy import openfoam from . import config as core_config from . import postprocess @@ -29,7 +30,7 @@ class UltimateRunner(object): # Configuration file self.config = config or core_config.default_config() - + # Database preparation self.database = Database(self.config["database"]) self.exec_id = None @@ -51,6 +52,24 @@ class UltimateRunner(object): # Parameters self.queue = [] + def _query_database(self, tableName: str, to_dict: bool = False): + tableName = tableName.lower() + get_row = { + "shape": self.database.getShape, + "mesh": self.database.getMesh, + "flowonephase": self.database.getFlowOnephase + }[tableName] + # query + args = ( + self.config.params["label"], + self.config.params["direction"], + self.config.params["alpha"], + self.exec_id, + to_dict + ) + + return get_row(*args) + def prepare_database(self): # create a row in each table for the current case # shape @@ -128,25 +147,16 @@ class UltimateRunner(object): def compute_shape(self): params = self.config.params - shapeParams = self.database.getShape( - params["label"], - params["direction"], - params["alpha"], - self.exec_id - ) + shapeParams = self._query_database("shape") logger.info("Computing shape for {} with direction = {} and alpha = {}".format( params["label"], params["direction"], params["alpha"] )) timer = utils.Timer() - # check - if ( - self.shapefile.exists() and - shapeParams.shapeStatus == "done" and - not self.config["overwrite"] - ): - logger.info("Shape exists. Skipping ...") + # check physical existence + if self.shapefile.exists() and self.config["overwrite"] is not True: + logger.info("Shape exists, skipping ...") return # @@ -187,25 +197,16 @@ class UltimateRunner(object): def compute_mesh(self): params = self.config.params - meshParams = self.database.getMesh( - params["label"], - params["direction"], - params["alpha"], - self.exec_id - ) + meshParams = self._query_database("mesh") logger.info("Computing mesh for {} with direction = {} and alpha = {}".format( params["label"], params["direction"], params["alpha"] )) timer = utils.Timer() - # check - if ( - self.meshfile.exists() and - meshParams.meshStatus == "done" and - not self.config["overwrite"] - ): - logger.info("Mesh exists. Skipping ...") + # check physical existence + if self.meshfile.exists() and self.config["overwrite"] is not True: + logger.info("Mesh exists, skipping ...") return elif not self.shapefile.exists(): @@ -246,21 +247,15 @@ class UltimateRunner(object): def compute_flow(self): params = self.config.params - query = ( - params["label"], - params["direction"], - params["alpha"], - self.exec_id - ) - flowParams = self.database.getFlowOnephase(*query) + flowParams = self._query_database("flowonephase") logger.info("Computing flow for {} with direction = {} and alpha = {}".format( params["label"], params["direction"], params["alpha"] )) timer = utils.Timer() - # check - if flowParams.flowStatus == "done" and not self.config["overwrite"]: + # check physical existence + if openfoam.FoamCase(path = self.casepath).is_converged() and not self.config["overwrite"]: logger.info("Solution exists. Skipping ...") return @@ -277,7 +272,7 @@ class UltimateRunner(object): self.database.csave(flowParams) # - flowParamsDict = self.database.getFlowOnephase(*query, to_dict = True) + flowParamsDict = self._query_database("flowonephase", to_dict = True) try: # load shape @@ -309,19 +304,15 @@ class UltimateRunner(object): def compute_postprocess(self): params = self.config.params - flowParams = self.database.getFlowOnephase( - params["label"], - params["direction"], - params["alpha"], - self.exec_id - ) + flowParams = self._query_database("flowonephase") logger.info("Computing post process for {} with direction = {} and alpha = {}".format( params["label"], params["direction"], params["alpha"] )) if flowParams.flowStatus == "done": - flowParams.flowRate = postprocess.flowRate("outlet", self.casepath) + if flowParams.flowRate is None and self.config["overwrite"] is not True: + flowParams.flowRate = postprocess.flowRate("outlet", self.casepath) self.database.csave(flowParams) @@ -335,15 +326,43 @@ class UltimateRunner(object): for current in stages: if current == "shape": + params = self._query_database("shape") + + # check database entry + if params.shapeStatus == "done" and self.config["overwrite"] is not True: + logger.info("Successful shape entry exists, skipping ...") + continue + self.compute_shape() if current == "mesh": + params = self._query_database("mesh") + + # check database entry + if params.meshStatus == "done" and self.config["overwrite"] is not True: + logger.info("Successful mesh entry exists, skipping ...") + continue + self.compute_mesh() if current == "flow": + params = self._query_database("flowonephase") + + # check database entry + if params.flowStatus == "done" and self.config["overwrite"] is not True: + logger.info("Successful flow entry exists, skipping ...") + continue + self.compute_flow() if current == "postProcess": + params = self._query_database("flowonephase") + + # check database entry + # if params.flowStatus == "done" and self.config["overwrite"] is not True: + # logger.info("Successful flow entry exists, skipping ...") + # continue + self.compute_postprocess() if current == stage or current == "all": diff --git a/anisotropy/database/database.py b/anisotropy/database/database.py index 875b4db..646554a 100644 --- a/anisotropy/database/database.py +++ b/anisotropy/database/database.py @@ -57,7 +57,7 @@ class Database(pw.SqliteDatabase): return self - def csave(self, table: pw.Model, tries: int = 100): + def csave(self, table: pw.Model, tries: int = 5000): """Try to save data from model to the database ignoring peewee.OperationalError. Usefull for concurrent processes. @@ -82,7 +82,12 @@ class Database(pw.SqliteDatabase): self.close() break - def getExecution(self, idn: int) -> tables.Execution | None: + def getExecution( + self, + idn: int, + to_dict: bool = False, + **kwargs + ) -> tables.Execution | None: """Get execution entry from database. :param idn: @@ -93,7 +98,11 @@ class Database(pw.SqliteDatabase): query = tables.Execution.select().where(tables.Execution.exec_id == idn) with self: - table = query.get() if query.exists() else None + if to_dict: + table = query.dicts().get() if query.exists() else None + + else: + table = query.get() if query.exists() else None return table @@ -116,6 +125,7 @@ class Database(pw.SqliteDatabase): direction: list[float] | ndarray = None, alpha: float = None, execution: int = None, + to_dict: bool = False, **kwargs ) -> tables.Shape | None: """Get shape entry from database. @@ -145,7 +155,11 @@ class Database(pw.SqliteDatabase): ) with self: - table = query.get() if query.exists() else None + if to_dict: + table = query.dicts().get() if query.exists() else None + + else: + table = query.get() if query.exists() else None return table @@ -155,6 +169,7 @@ class Database(pw.SqliteDatabase): direction: list[float] | ndarray = None, alpha: float = None, execution: int = None, + to_dict: bool = False, **kwargs ) -> tables.Mesh | None: """Get mesh entry from database. @@ -185,7 +200,11 @@ class Database(pw.SqliteDatabase): ) with self: - table = query.get() if query.exists() else None + if to_dict: + table = query.dicts().get() if query.exists() else None + + else: + table = query.get() if query.exists() else None return table diff --git a/anisotropy/gui/layouts/visualization.py b/anisotropy/gui/layouts/visualization.py index 98f6ba0..81cd3f3 100644 --- a/anisotropy/gui/layouts/visualization.py +++ b/anisotropy/gui/layouts/visualization.py @@ -231,7 +231,7 @@ layout = html.Div([ ] ) def plotDraw(clicks, execution, structure, direction, data): - from peewee import JOIN + import peewee as pw from anisotropy.database import Database, tables import json from pandas import DataFrame @@ -263,21 +263,33 @@ def plotDraw(clicks, execution, structure, direction, data): else: select = (tables.Shape.alpha, column) - query = ( + """query = ( tables.Shape .select(*select) .join(tables.Execution, JOIN.LEFT_OUTER) .switch(tables.Shape) .join(tables.Mesh, JOIN.LEFT_OUTER) + #.switch(tables.Shape) + .join(tables.FlowOnephase, JOIN.LEFT_OUTER) .switch(tables.Shape) - # .join(tables.FlowOnephase, JOIN.LEFT_OUTER) - # .switch(tables.Shape) .where( tables.Shape.exec_id == execution, tables.Shape.label == structure, ) - ) + )""" + query = model.select(*select) + idn = db.tables.index(model) + for table in reversed(db.tables[ :idn]): + query = query.join(table, pw.JOIN.LEFT_OUTER) + + query = query.switch(tables.Shape) + query = query.where( + tables.Shape.exec_id == execution, + tables.Shape.label == structure, + ) + query = query.order_by(tables.Shape.alpha) + if not direction == "all": query = query.where(tables.Shape.direction == json.loads(direction)) diff --git a/anisotropy/openfoam/case.py b/anisotropy/openfoam/case.py index 415a8fd..1bf0f3e 100644 --- a/anisotropy/openfoam/case.py +++ b/anisotropy/openfoam/case.py @@ -6,6 +6,8 @@ import os import shutil import re import pathlib +import io +from lz.reversal import reverse as lz_reverse from . import FoamFile @@ -72,23 +74,23 @@ class FoamCase(object): path = pathlib.Path(path or self.path or "") for file in self._files: - path /= ( + path_ = path / ( file.location + "/" + file.object if file.location else file.object ) - file.read(path.resolve()) + file.read(path_.resolve()) def remove(self, path: str = None): path = pathlib.Path(path or self.path or "") for file in self._files: - path /= ( + path_ = path / ( file.location + "/" + file.object if file.location else file.object ) - file.remove(path.resolve()) + file.remove(path_.resolve()) def clean(self, included: list = ["0", "constant", "system"]): regxs = [ @@ -105,10 +107,16 @@ class FoamCase(object): for root, dirs, files in os.walk(os.path.abspath("")): for _dir in dirs: - excluded += [ os.path.join(root, _dir) for regx in regxs if re.match(regx, _dir) ] + excluded += [ + os.path.join(root, _dir) + for regx in regxs if re.match(regx, _dir) + ] for file in files: - excluded += [ os.path.join(root, file) for regx in regxs if re.match(regx, file) ] + excluded += [ + os.path.join(root, file) + for regx in regxs if re.match(regx, file) + ] for file in excluded: if os.path.split(file)[1] not in included and os.path.exists(file): @@ -128,3 +136,39 @@ class FoamCase(object): path = pathlib.Path(self._backpath or "").resolve() os.chdir(path) + + def is_converged(self, path: str = None) -> None | bool: + path = pathlib.Path(path or self.path or "").resolve() + controlDict = FoamFile() + + if controlDict.exists(path / "system" / "controlDict"): + controlDict.read(path / "system" / "controlDict") + + else: + return None + + application = controlDict.get("application") + + if application is None: + return None + + logfile = ( + path / f"{ application }.log" + if (path / f"{ application }.log").exists() else (path / f"log.{ application }") + ) + status = False + + if logfile.exists(): + with open(logfile, "r") as infile: + limit = 30 + + for line in lz_reverse(infile, batch_size = io.DEFAULT_BUFFER_SIZE): + if not line.find("End") < 0: + status = True + + if limit <= 0: + status = False + + limit -= 1 + + return status diff --git a/anisotropy/openfoam/conversion.py b/anisotropy/openfoam/conversion.py index c3e0307..17688ef 100644 --- a/anisotropy/openfoam/conversion.py +++ b/anisotropy/openfoam/conversion.py @@ -22,6 +22,10 @@ def read_foamfile(filename: str) -> tuple[dict, dict]: header = ppf.header or {} content = ppf.content or {} + if type(content) == dict: + if content.get("internalField"): + content["internalField"] = np.asarray(content["internalField"]) + return header, content diff --git a/anisotropy/openfoam/file.py b/anisotropy/openfoam/file.py index 0e8b068..d952265 100644 --- a/anisotropy/openfoam/file.py +++ b/anisotropy/openfoam/file.py @@ -68,11 +68,17 @@ class FoamFile(object): @property def location(self) -> str: - return self.header.get("location").replace('"', "") + return ( + self.header.get("location").replace('"', "") + if self.header.get("location") else None + ) def __getitem__(self, key): return self.content[key] + def get(self, key, default = None): + return self.content.get(key, default) + def __setitem__(self, key, value): self.content[key] = value @@ -99,6 +105,15 @@ class FoamFile(object): return FoamCase([ self, file ]) + def exists(self, filename: str = None) -> bool: + filename = filename or ( + self.location + "/" + self.object + if self.location else self.object + ) + path = pathlib.Path(filename).resolve() + + return path.exists() and path.is_file() + def read(self, filename: str = None): """Read a FoamFile. @@ -108,8 +123,8 @@ class FoamFile(object): :return: Self. """ - filename = ( - filename or self.location + "/" + self.object + filename = filename or ( + self.location + "/" + self.object if self.location else self.object ) path = pathlib.Path(filename).resolve() @@ -131,8 +146,8 @@ class FoamFile(object): Path to the file. If None, use location and object from header with the current working directory. """ - filename = ( - filename or self.location + "/" + self.object + filename = filename or ( + self.location + "/" + self.object if self.location else self.object ) path = pathlib.Path(filename).resolve() @@ -146,8 +161,8 @@ class FoamFile(object): Path to the file. If None, use location and object from header with the current working directory. """ - filename = ( - filename or self.location + "/" + self.object + filename = filename or ( + self.location + "/" + self.object if self.location else self.object ) path = pathlib.Path(filename).resolve()