From a4e9a8f8dc6cdb1979afd7fc72b052cb232cffe1 Mon Sep 17 00:00:00 2001 From: L-Nafaryus Date: Mon, 24 Jan 2022 17:50:57 +0500 Subject: [PATCH] Mod: database is now stable. Improved convenience and documention. --- .vscode/settings.json | 2 +- anisotropy/database/__init__.py | 16 ++- anisotropy/database/database.py | 238 ++++++++++++++++++++++++++++++++ anisotropy/database/db.py | 140 ------------------- anisotropy/database/models.py | 108 --------------- anisotropy/database/tables.py | 102 ++++++++++++++ anisotropy/database/utils.py | 115 +++++++-------- 7 files changed, 399 insertions(+), 322 deletions(-) create mode 100644 anisotropy/database/database.py delete mode 100644 anisotropy/database/db.py delete mode 100644 anisotropy/database/models.py create mode 100644 anisotropy/database/tables.py diff --git a/.vscode/settings.json b/.vscode/settings.json index ee48e86..424f93f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,5 @@ "python.linting.enabled": true, "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--ignore=E402,E251,E501,E201,E202,W293,W291,W504", "--verbose"] + "python.linting.flake8Args": ["--ignore=E402,E251,E501,E201,E202,W293,W291,W504,E203", "--verbose"] } \ No newline at end of file diff --git a/anisotropy/database/__init__.py b/anisotropy/database/__init__.py index 8540ec0..de89038 100644 --- a/anisotropy/database/__init__.py +++ b/anisotropy/database/__init__.py @@ -1,11 +1,13 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -from .models import __models__ -from .db import Database +from . import utils +from . import tables -class tables: - pass +from .database import Database -for model in __models__: - setattr(tables, model.__name__, model) \ No newline at end of file + +__all__ = [ + "utils", + "tables", + "Database" +] diff --git a/anisotropy/database/database.py b/anisotropy/database/database.py new file mode 100644 index 0000000..48da9b1 --- /dev/null +++ b/anisotropy/database/database.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +from __future__ import annotations +from numpy import ndarray + +import peewee as pw +import pathlib +import time + +from . import tables + + +class Database(pw.SqliteDatabase): + def __init__(self, *args, **kwargs): + """A Database object contains SQLite database with convient + properties and methods + """ + self.filepath = kwargs.get("path", None) + self.pragmas_ = kwargs.get("pragmas", { "foreign_keys": 1, "journal_mode": "wal" }) + self.field_types_ = kwargs.get("field_types", { "list": "text" }) + self.autoconnect_ = kwargs.get("autoconnect", False) + + pw.SqliteDatabase.__init__( + self, + None, + pragmas = self.pragmas_, + field_types = self.field_types_, + autoconnect = self.autoconnect_ + ) + + if self.filepath: + self.setup() + + @property + def tables(self) -> list: + """Return all tables as list. + """ + return [ tables.__dict__[table] for table in tables.__all__ ] + + def setup(self, filename: str = None): + """Initialize database and create tables. + + :param filename: + Path to the file. + :return: + Self. + """ + self.filepath = pathlib.Path(filename or self.filepath).resolve() + self.init( + self.filepath, + pragmas = self.pragmas_ + ) + tables.database_proxy.initialize(self) + + with self: + self.create_tables(self.tables) + + return self + + def csave(self, table: pw.Model, tries: int = 100): + """Try to save data from model to the database ignoring + peewee.OperationalError. Usefull for concurrent processes. + + :param table: + Table to save. + :param tries: + Number of tries. Falling to sleep for 1 second if database + is locked. + """ + while tries >= 0: + if self.is_closed(): + self.connect() + + try: + table.save() + + except pw.OperationalError: + tries -= 1 + time.sleep(1) + + else: + self.close() + break + + def getExecution(self, idn: int) -> tables.Execution | None: + """Get execution entry from database. + + :param idn: + Index of the execution. + :return: + If entry is found returns Model instance else None. + """ + query = tables.Execution.select().where(tables.Execution.exec_id == idn) + + with self: + table = query.get() if query.exists() else None + + return table + + def getLatest(self) -> tables.Execution | None: + """Get latest execution entry from database. + + :return: + If entry is found returns Model instance else None. + """ + query = tables.Execution.select() + + with self: + table = query[-1] if query.exists() else None + + return table + + def getShape( + self, + label: str = None, + direction: list[float] | ndarray = None, + alpha: float = None, + execution: int = None, + **kwargs + ) -> tables.Shape | None: + """Get shape entry from database. + + :param label: + Label of the shape. + :param direction: + Array of floats represents direction vector. + :param alpha: + Spheres overlap parameter. + :param execution: + Index of the execution. If None, use latest. + :return: + If entry is found returns Model instance else None. + """ + execution = execution or self.getLatest() + query = ( + tables.Shape + .select() + .join(tables.Execution, pw.JOIN.LEFT_OUTER) + .where( + tables.Execution.exec_id == execution, + tables.Shape.label == label, + tables.Shape.direction == direction, + tables.Shape.alpha == alpha + ) + ) + + with self: + table = query.get() if query.exists() else None + + return table + + def getMesh( + self, + label: str = None, + direction: list[float] | ndarray = None, + alpha: float = None, + execution: int = None, + **kwargs + ) -> tables.Mesh | None: + """Get mesh entry from database. + + :param label: + Label of the shape. + :param direction: + Array of floats represents direction vector. + :param alpha: + Spheres overlap parameter. + :param execution: + Index of the execution. If None, use latest. + :return: + If entry is found returns Model instance else None. + """ + execution = execution or self.getLatest() + query = ( + tables.Mesh + .select() + .join(tables.Shape, pw.JOIN.LEFT_OUTER) + .join(tables.Execution, pw.JOIN.LEFT_OUTER) + .where( + tables.Execution.exec_id == execution, + tables.Shape.label == label, + tables.Shape.direction == direction, + tables.Shape.alpha == alpha + ) + ) + + with self: + table = query.get() if query.exists() else None + + return table + + def getFlowOnephase( + self, + label: str = None, + direction: list[float] | ndarray = None, + alpha: float = None, + execution: int = None, + to_dict: bool = False, + **kwargs + ) -> tables.Mesh | dict | None: + """Get one phase flow entry from database. + + :param label: + Label of the shape. + :param direction: + Array of floats represents direction vector. + :param alpha: + Spheres overlap parameter. + :param execution: + Index of the execution. If None, use latest. + :param to_dict: + If True, convert result to dict. + :return: + If entry is found returns Model instance or dict else None. + """ + execution = execution or self.getLatest() + query = ( + tables.FlowOnephase + .select() + .join(tables.Mesh, pw.JOIN.LEFT_OUTER) + .join(tables.Shape, pw.JOIN.LEFT_OUTER) + .join(tables.Execution, pw.JOIN.LEFT_OUTER) + .where( + tables.Execution.exec_id == execution, + tables.Shape.label == label, + tables.Shape.direction == direction, + tables.Shape.alpha == alpha + ) + ) + + with self: + if to_dict: + table = query.dicts().get() if query.exists() else None + + else: + table = query.get() if query.exists() else None + + return table diff --git a/anisotropy/database/db.py b/anisotropy/database/db.py deleted file mode 100644 index d8b9a07..0000000 --- a/anisotropy/database/db.py +++ /dev/null @@ -1,140 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of anisotropy. -# License: GNU GPL version 3, see the file "LICENSE" for details. - -import os -from peewee import SqliteDatabase, JOIN, OperationalError -from . import models - -class Database(SqliteDatabase): - def __init__(self, *args, **kwargs): - self.filepath = kwargs.get("path", None) - self.pragmas_ = kwargs.get("pragmas", { "foreign_keys": 1, "journal_mode": "wal" }) - self.field_types_ = kwargs.get("field_types", { "list": "text" }) - self.autoconnect_ = kwargs.get("autoconnect", False) - - SqliteDatabase.__init__( - self, - None, - pragmas = self.pragmas_, - field_types = self.field_types_, - autoconnect = self.autoconnect_ - ) - - if self.filepath: - self.setup() - - @property - def tables(self): - return models.__models__ - - def setup(self, filename: str = None): - #if not self.filepath: - self.filepath = os.path.abspath(filename or self.filepath) - self.init( - self.filepath, - pragmas = self.pragmas_, - #field_types = self.field_types_, - #autoconnect = self.autoconnect_ - ) - models.__database_proxy__.initialize(self) - - with self: - self.create_tables(self.tables) - - def csave(self, table, tries: int = 100): - while tries >= 0: - if self.is_closed(): - self.connect() - - try: - table.save() - - except OperationalError as e: - logger.debug(e) - tries -= 1 - time.sleep(1) - - else: - self.close() - break - - def getExecution(self, idn): - query = models.Execution.select().where(models.Execution.exec_id == idn) - - with self: - table = query.get() if query.exists() else None - - return table - - def getLatest(self): - query = models.Execution.select() - - with self: - table = query[-1] if query.exists() else None - - return table - - def getShape(self, label = None, direction = None, alpha = None, execution = None, **kwargs): - execution = execution or self.getLatest() - query = ( - models.Shape - .select() - .join(models.Execution, JOIN.LEFT_OUTER) - .where( - models.Execution.exec_id == execution, - models.Shape.label == label, - models.Shape.direction == direction, - models.Shape.alpha == alpha - ) - ) - - with self: - table = query.get() if query.exists() else None - - return table - - def getMesh(self, label = None, direction = None, alpha = None, execution = None, **kwargs): - execution = execution or self.getLatest() - query = ( - models.Mesh - .select() - .join(models.Shape, JOIN.LEFT_OUTER) - .join(models.Execution, JOIN.LEFT_OUTER) - .where( - models.Execution.exec_id == execution, - models.Shape.label == label, - models.Shape.direction == direction, - models.Shape.alpha == alpha - ) - ) - - with self: - table = query.get() if query.exists() else None - - return table - - def getFlowOnephase(self, label = None, direction = None, alpha = None, execution = None, to_dict = False, **kwargs): - execution = execution or self.getLatest() - query = ( - models.FlowOnephase - .select() - .join(models.Mesh, JOIN.LEFT_OUTER) - .join(models.Shape, JOIN.LEFT_OUTER) - .join(models.Execution, JOIN.LEFT_OUTER) - .where( - models.Execution.exec_id == execution, - models.Shape.label == label, - models.Shape.direction == direction, - models.Shape.alpha == alpha - ) - ) - - with self: - if to_dict: - table = query.dicts().get() if query.exists() else None - - else: - table = query.get() if query.exists() else None - - return table diff --git a/anisotropy/database/models.py b/anisotropy/database/models.py deleted file mode 100644 index 616f1a5..0000000 --- a/anisotropy/database/models.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of anisotropy. -# License: GNU GPL version 3, see the file "LICENSE" for details. - -from peewee import ( - SqliteDatabase, JOIN, - Model, Field, - AutoField, ForeignKeyField, - TextField, FloatField, - IntegerField, BooleanField, - TimeField, DateTimeField, Proxy -) -from .utils import JSONField - -__database_proxy__ = Proxy() - -class Execution(Model): - exec_id = AutoField() - - date = DateTimeField() - executionTime = TimeField(null = True) - - class Meta: - database = __database_proxy__ - table_name = "executions" - - -class Shape(Model): - shape_id = AutoField() - exec_id = ForeignKeyField(Execution, backref = "executions", on_delete = "CASCADE") - - shapeStatus = TextField(null = True, default = "idle") - shapeExecutionTime = TimeField(null = True) - - label = TextField(null = True) - direction = JSONField(null = True) - alpha = FloatField(null = True) - - r0 = FloatField(null = True) - L = FloatField(null = True) - radius = FloatField(null = True) - - filletsEnabled = BooleanField(null = True) - fillets = FloatField(null = True) - - volumeCell = FloatField(null = True) - volume = FloatField(null = True) - porosity = FloatField(null = True) - - class Meta: - database = __database_proxy__ - table_name = "shapes" - #depends_on = Execution - - -class Mesh(Model): - mesh_id = AutoField() - shape_id = ForeignKeyField(Shape, backref = "shapes", on_delete = "CASCADE") - - meshStatus = TextField(null = True, default = "idle") - meshExecutionTime = TimeField(null = True) - - elements = IntegerField(null = True) - edges = IntegerField(null = True) - faces = IntegerField(null = True) - volumes = IntegerField(null = True) - tetrahedrons = IntegerField(null = True) - prisms = IntegerField(null = True) - pyramids = IntegerField(null = True) - - - class Meta: - database = __database_proxy__ - table_name = "meshes" - #depends_on = Execution - - -class FlowOnephase(Model): - flow_id = AutoField() - mesh_id = ForeignKeyField(Mesh, backref = "meshes", on_delete = "CASCADE") - - flowStatus = TextField(null = True, default = "idle") - flowExecutionTime = TimeField(null = True) - - pressureInlet = FloatField(null = True) - pressureOutlet = FloatField(null = True) - pressureInternal = FloatField(null = True) - velocityInlet = JSONField(null = True) - velocityOutlet = JSONField(null = True) - velocityInternal = JSONField(null = True) - viscosity = FloatField(null = True) - viscosityKinematic = FloatField(null = True) - density = FloatField(null = True) - flowRate = FloatField(null = True) - permeability = FloatField(null = True) - - class Meta: - database = __database_proxy__ - table_name = "flows" - #depends_on = Execution - - -__models__ = [ - Execution, - Shape, - Mesh, - FlowOnephase -] \ No newline at end of file diff --git a/anisotropy/database/tables.py b/anisotropy/database/tables.py new file mode 100644 index 0000000..58771fe --- /dev/null +++ b/anisotropy/database/tables.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +import peewee as pw + +from . import utils + + +# proxy, assign database later +database_proxy = pw.Proxy() + + +class Execution(pw.Model): + exec_id = pw.AutoField() + + date = pw.DateTimeField() + executionTime = pw.TimeField(null = True) + + class Meta: + database = database_proxy + table_name = "executions" + + +class Shape(pw.Model): + shape_id = pw.AutoField() + exec_id = pw.ForeignKeyField(Execution, backref = "executions", on_delete = "CASCADE") + + shapeStatus = pw.TextField(null = True, default = "idle") + shapeExecutionTime = pw.TimeField(null = True) + + label = pw.TextField(null = True) + direction = utils.JSONField(null = True) + alpha = pw.FloatField(null = True) + + r0 = pw.FloatField(null = True) + L = pw.FloatField(null = True) + radius = pw.FloatField(null = True) + + filletsEnabled = pw.BooleanField(null = True) + fillets = pw.FloatField(null = True) + + volumeCell = pw.FloatField(null = True) + volume = pw.FloatField(null = True) + porosity = pw.FloatField(null = True) + + class Meta: + database = database_proxy + table_name = "shapes" + # depends_on = Execution + + +class Mesh(pw.Model): + mesh_id = pw.AutoField() + shape_id = pw.ForeignKeyField(Shape, backref = "shapes", on_delete = "CASCADE") + + meshStatus = pw.TextField(null = True, default = "idle") + meshExecutionTime = pw.TimeField(null = True) + + elements = pw.IntegerField(null = True) + edges = pw.IntegerField(null = True) + faces = pw.IntegerField(null = True) + volumes = pw.IntegerField(null = True) + tetrahedrons = pw.IntegerField(null = True) + prisms = pw.IntegerField(null = True) + pyramids = pw.IntegerField(null = True) + + class Meta: + database = database_proxy + table_name = "meshes" + # depends_on = Execution + + +class FlowOnephase(pw.Model): + flow_id = pw.AutoField() + mesh_id = pw.ForeignKeyField(Mesh, backref = "meshes", on_delete = "CASCADE") + + flowStatus = pw.TextField(null = True, default = "idle") + flowExecutionTime = pw.TimeField(null = True) + + pressureInlet = pw.FloatField(null = True) + pressureOutlet = pw.FloatField(null = True) + pressureInternal = pw.FloatField(null = True) + velocityInlet = utils.JSONField(null = True) + velocityOutlet = utils.JSONField(null = True) + velocityInternal = utils.JSONField(null = True) + viscosity = pw.FloatField(null = True) + viscosityKinematic = pw.FloatField(null = True) + density = pw.FloatField(null = True) + flowRate = pw.FloatField(null = True) + permeability = pw.FloatField(null = True) + + class Meta: + database = database_proxy + table_name = "flows" + # depends_on = Execution + + +__all__ = [ + "Execution", + "Shape", + "Mesh", + "FlowOnephase" +] diff --git a/anisotropy/database/utils.py b/anisotropy/database/utils.py index a1f705c..3483667 100644 --- a/anisotropy/database/utils.py +++ b/anisotropy/database/utils.py @@ -1,22 +1,12 @@ # -*- coding: utf-8 -*- -# This file is part of anisotropy. -# License: GNU GPL version 3, see the file "LICENSE" for details. -from peewee import ( - TextField, - ColumnBase, - Value, - fn, - Node, - Expression, - OP, - Field -) -import json from numpy import ndarray +import peewee as pw +import json -class ListField(TextField): + +class ListField(pw.TextField): field_type = "list" def db_value(self, value): @@ -29,88 +19,78 @@ class ListField(TextField): try: pval.append(float(entry)) - except: + except Exception: pval.append(entry.strip().replace("'", "")) return pval -""" -class JSONField(TextField): - # TODO: fix double quotes when use __eq__ in 'where' method - field_type = "TEXT" - def db_value(self, value): - if isinstance(value, ndarray): - formatted = list(value) - - else: - formatted = value - - return json.dumps(formatted) - - def python_value(self, value): - if value is not None: - return json.loads(value) -""" - -class JSONPath(ColumnBase): - def __init__(self, field, path=None): +class JSONPath(pw.ColumnBase): + def __init__(self, field, path = None): super(JSONPath, self).__init__() + self._field = field self._path = path or () @property def path(self): - return Value('$%s' % ''.join(self._path)) + return pw.Value('$%s' % ''.join(self._path)) def __getitem__(self, idx): if isinstance(idx, int): item = '[%s]' % idx + else: item = '.%s' % idx + return JSONPath(self._field, self._path + (item,)) - def set(self, value, as_json=None): + def set(self, value, as_json = None): if as_json or isinstance(value, (list, dict)): - value = fn.json(self._field._json_dumps(value)) - return fn.json_set(self._field, self.path, value) + value = pw.fn.json(self._field._json_dumps(value)) + + return pw.fn.json_set(self._field, self.path, value) def update(self, value): - return self.set(fn.json_patch(self, self._field._json_dumps(value))) + return self.set(pw.fn.json_patch(self, self._field._json_dumps(value))) def remove(self): - return fn.json_remove(self._field, self.path) + return pw.fn.json_remove(self._field, self.path) def json_type(self): - return fn.json_type(self._field, self.path) + return pw.fn.json_type(self._field, self.path) def length(self): - return fn.json_array_length(self._field, self.path) + return pw.fn.json_array_length(self._field, self.path) def children(self): - return fn.json_each(self._field, self.path) + return pw.fn.json_each(self._field, self.path) def tree(self): - return fn.json_tree(self._field, self.path) + return pw.fn.json_tree(self._field, self.path) def __sql__(self, ctx): - return ctx.sql(fn.json_extract(self._field, self.path) - if self._path else self._field) + return ctx.sql( + pw.fn.json_extract(self._field, self.path) + if self._path else self._field + ) -class JSONField(TextField): +class JSONField(pw.TextField): field_type = 'TEXT' unpack = False - def __init__(self, json_dumps=None, json_loads=None, **kwargs): + def __init__(self, json_dumps = None, json_loads = None, **kwargs): + super(JSONField, self).__init__(**kwargs) + self._json_dumps = json_dumps or json.dumps self._json_loads = json_loads or json.loads - super(JSONField, self).__init__(**kwargs) def python_value(self, value): if value is not None: try: return json.loads(value) + except (TypeError, ValueError): return value @@ -119,28 +99,32 @@ class JSONField(TextField): if isinstance(value, ndarray): value = list(value) - if not isinstance(value, Node): + if not isinstance(value, pw.Node): value = json.dumps(value) + return value def _e(op): def inner(self, rhs): if isinstance(rhs, (list, dict)): - rhs = Value(rhs, converter=self.db_value, unpack=False) - return Expression(self, op, rhs) + rhs = pw.Value(rhs, converter = self.db_value, unpack = False) + + return pw.Expression(self, op, rhs) + return inner - __eq__ = _e(OP.EQ) - __ne__ = _e(OP.NE) - __gt__ = _e(OP.GT) - __ge__ = _e(OP.GTE) - __lt__ = _e(OP.LT) - __le__ = _e(OP.LTE) - __hash__ = Field.__hash__ + + __eq__ = _e(pw.OP.EQ) + __ne__ = _e(pw.OP.NE) + __gt__ = _e(pw.OP.GT) + __ge__ = _e(pw.OP.GTE) + __lt__ = _e(pw.OP.LT) + __le__ = _e(pw.OP.LTE) + __hash__ = pw.Field.__hash__ def __getitem__(self, item): return JSONPath(self)[item] - def set(self, value, as_json=None): + def set(self, value, as_json = None): return JSONPath(self).set(value, as_json) def update(self, data): @@ -150,10 +134,10 @@ class JSONField(TextField): return JSONPath(self).remove() def json_type(self): - return fn.json_type(self) + return pw.fn.json_type(self) def length(self): - return fn.json_array_length(self) + return pw.fn.json_array_length(self) def children(self): """ @@ -170,8 +154,7 @@ class JSONField(TextField): json JSON hidden (1st input parameter to function) root TEXT hidden (2nd input parameter, path at which to start) """ - return fn.json_each(self) + return pw.fn.json_each(self) def tree(self): - return fn.json_tree(self) - + return pw.fn.json_tree(self)