Mod: database is now stable.

Improved convenience and documention.
This commit is contained in:
L-Nafaryus 2022-01-24 17:50:57 +05:00
parent 00f7279c6d
commit a4e9a8f8dc
7 changed files with 399 additions and 322 deletions

View File

@ -8,5 +8,5 @@
"python.linting.enabled": true,
"python.linting.pylintEnabled": false,
"python.linting.flake8Enabled": true,
"python.linting.flake8Args": ["--ignore=E402,E251,E501,E201,E202,W293,W291,W504", "--verbose"]
"python.linting.flake8Args": ["--ignore=E402,E251,E501,E201,E202,W293,W291,W504,E203", "--verbose"]
}

View File

@ -1,11 +1,13 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from .models import __models__
from .db import Database
from . import utils
from . import tables
class tables:
pass
from .database import Database
for model in __models__:
setattr(tables, model.__name__, model)
__all__ = [
"utils",
"tables",
"Database"
]

View File

@ -0,0 +1,238 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from numpy import ndarray
import peewee as pw
import pathlib
import time
from . import tables
class Database(pw.SqliteDatabase):
def __init__(self, *args, **kwargs):
"""A Database object contains SQLite database with convient
properties and methods
"""
self.filepath = kwargs.get("path", None)
self.pragmas_ = kwargs.get("pragmas", { "foreign_keys": 1, "journal_mode": "wal" })
self.field_types_ = kwargs.get("field_types", { "list": "text" })
self.autoconnect_ = kwargs.get("autoconnect", False)
pw.SqliteDatabase.__init__(
self,
None,
pragmas = self.pragmas_,
field_types = self.field_types_,
autoconnect = self.autoconnect_
)
if self.filepath:
self.setup()
@property
def tables(self) -> list:
"""Return all tables as list.
"""
return [ tables.__dict__[table] for table in tables.__all__ ]
def setup(self, filename: str = None):
"""Initialize database and create tables.
:param filename:
Path to the file.
:return:
Self.
"""
self.filepath = pathlib.Path(filename or self.filepath).resolve()
self.init(
self.filepath,
pragmas = self.pragmas_
)
tables.database_proxy.initialize(self)
with self:
self.create_tables(self.tables)
return self
def csave(self, table: pw.Model, tries: int = 100):
"""Try to save data from model to the database ignoring
peewee.OperationalError. Usefull for concurrent processes.
:param table:
Table to save.
:param tries:
Number of tries. Falling to sleep for 1 second if database
is locked.
"""
while tries >= 0:
if self.is_closed():
self.connect()
try:
table.save()
except pw.OperationalError:
tries -= 1
time.sleep(1)
else:
self.close()
break
def getExecution(self, idn: int) -> tables.Execution | None:
"""Get execution entry from database.
:param idn:
Index of the execution.
:return:
If entry is found returns Model instance else None.
"""
query = tables.Execution.select().where(tables.Execution.exec_id == idn)
with self:
table = query.get() if query.exists() else None
return table
def getLatest(self) -> tables.Execution | None:
"""Get latest execution entry from database.
:return:
If entry is found returns Model instance else None.
"""
query = tables.Execution.select()
with self:
table = query[-1] if query.exists() else None
return table
def getShape(
self,
label: str = None,
direction: list[float] | ndarray = None,
alpha: float = None,
execution: int = None,
**kwargs
) -> tables.Shape | None:
"""Get shape entry from database.
:param label:
Label of the shape.
:param direction:
Array of floats represents direction vector.
:param alpha:
Spheres overlap parameter.
:param execution:
Index of the execution. If None, use latest.
:return:
If entry is found returns Model instance else None.
"""
execution = execution or self.getLatest()
query = (
tables.Shape
.select()
.join(tables.Execution, pw.JOIN.LEFT_OUTER)
.where(
tables.Execution.exec_id == execution,
tables.Shape.label == label,
tables.Shape.direction == direction,
tables.Shape.alpha == alpha
)
)
with self:
table = query.get() if query.exists() else None
return table
def getMesh(
self,
label: str = None,
direction: list[float] | ndarray = None,
alpha: float = None,
execution: int = None,
**kwargs
) -> tables.Mesh | None:
"""Get mesh entry from database.
:param label:
Label of the shape.
:param direction:
Array of floats represents direction vector.
:param alpha:
Spheres overlap parameter.
:param execution:
Index of the execution. If None, use latest.
:return:
If entry is found returns Model instance else None.
"""
execution = execution or self.getLatest()
query = (
tables.Mesh
.select()
.join(tables.Shape, pw.JOIN.LEFT_OUTER)
.join(tables.Execution, pw.JOIN.LEFT_OUTER)
.where(
tables.Execution.exec_id == execution,
tables.Shape.label == label,
tables.Shape.direction == direction,
tables.Shape.alpha == alpha
)
)
with self:
table = query.get() if query.exists() else None
return table
def getFlowOnephase(
self,
label: str = None,
direction: list[float] | ndarray = None,
alpha: float = None,
execution: int = None,
to_dict: bool = False,
**kwargs
) -> tables.Mesh | dict | None:
"""Get one phase flow entry from database.
:param label:
Label of the shape.
:param direction:
Array of floats represents direction vector.
:param alpha:
Spheres overlap parameter.
:param execution:
Index of the execution. If None, use latest.
:param to_dict:
If True, convert result to dict.
:return:
If entry is found returns Model instance or dict else None.
"""
execution = execution or self.getLatest()
query = (
tables.FlowOnephase
.select()
.join(tables.Mesh, pw.JOIN.LEFT_OUTER)
.join(tables.Shape, pw.JOIN.LEFT_OUTER)
.join(tables.Execution, pw.JOIN.LEFT_OUTER)
.where(
tables.Execution.exec_id == execution,
tables.Shape.label == label,
tables.Shape.direction == direction,
tables.Shape.alpha == alpha
)
)
with self:
if to_dict:
table = query.dicts().get() if query.exists() else None
else:
table = query.get() if query.exists() else None
return table

View File

@ -1,140 +0,0 @@
# -*- coding: utf-8 -*-
# This file is part of anisotropy.
# License: GNU GPL version 3, see the file "LICENSE" for details.
import os
from peewee import SqliteDatabase, JOIN, OperationalError
from . import models
class Database(SqliteDatabase):
def __init__(self, *args, **kwargs):
self.filepath = kwargs.get("path", None)
self.pragmas_ = kwargs.get("pragmas", { "foreign_keys": 1, "journal_mode": "wal" })
self.field_types_ = kwargs.get("field_types", { "list": "text" })
self.autoconnect_ = kwargs.get("autoconnect", False)
SqliteDatabase.__init__(
self,
None,
pragmas = self.pragmas_,
field_types = self.field_types_,
autoconnect = self.autoconnect_
)
if self.filepath:
self.setup()
@property
def tables(self):
return models.__models__
def setup(self, filename: str = None):
#if not self.filepath:
self.filepath = os.path.abspath(filename or self.filepath)
self.init(
self.filepath,
pragmas = self.pragmas_,
#field_types = self.field_types_,
#autoconnect = self.autoconnect_
)
models.__database_proxy__.initialize(self)
with self:
self.create_tables(self.tables)
def csave(self, table, tries: int = 100):
while tries >= 0:
if self.is_closed():
self.connect()
try:
table.save()
except OperationalError as e:
logger.debug(e)
tries -= 1
time.sleep(1)
else:
self.close()
break
def getExecution(self, idn):
query = models.Execution.select().where(models.Execution.exec_id == idn)
with self:
table = query.get() if query.exists() else None
return table
def getLatest(self):
query = models.Execution.select()
with self:
table = query[-1] if query.exists() else None
return table
def getShape(self, label = None, direction = None, alpha = None, execution = None, **kwargs):
execution = execution or self.getLatest()
query = (
models.Shape
.select()
.join(models.Execution, JOIN.LEFT_OUTER)
.where(
models.Execution.exec_id == execution,
models.Shape.label == label,
models.Shape.direction == direction,
models.Shape.alpha == alpha
)
)
with self:
table = query.get() if query.exists() else None
return table
def getMesh(self, label = None, direction = None, alpha = None, execution = None, **kwargs):
execution = execution or self.getLatest()
query = (
models.Mesh
.select()
.join(models.Shape, JOIN.LEFT_OUTER)
.join(models.Execution, JOIN.LEFT_OUTER)
.where(
models.Execution.exec_id == execution,
models.Shape.label == label,
models.Shape.direction == direction,
models.Shape.alpha == alpha
)
)
with self:
table = query.get() if query.exists() else None
return table
def getFlowOnephase(self, label = None, direction = None, alpha = None, execution = None, to_dict = False, **kwargs):
execution = execution or self.getLatest()
query = (
models.FlowOnephase
.select()
.join(models.Mesh, JOIN.LEFT_OUTER)
.join(models.Shape, JOIN.LEFT_OUTER)
.join(models.Execution, JOIN.LEFT_OUTER)
.where(
models.Execution.exec_id == execution,
models.Shape.label == label,
models.Shape.direction == direction,
models.Shape.alpha == alpha
)
)
with self:
if to_dict:
table = query.dicts().get() if query.exists() else None
else:
table = query.get() if query.exists() else None
return table

View File

@ -1,108 +0,0 @@
# -*- coding: utf-8 -*-
# This file is part of anisotropy.
# License: GNU GPL version 3, see the file "LICENSE" for details.
from peewee import (
SqliteDatabase, JOIN,
Model, Field,
AutoField, ForeignKeyField,
TextField, FloatField,
IntegerField, BooleanField,
TimeField, DateTimeField, Proxy
)
from .utils import JSONField
__database_proxy__ = Proxy()
class Execution(Model):
exec_id = AutoField()
date = DateTimeField()
executionTime = TimeField(null = True)
class Meta:
database = __database_proxy__
table_name = "executions"
class Shape(Model):
shape_id = AutoField()
exec_id = ForeignKeyField(Execution, backref = "executions", on_delete = "CASCADE")
shapeStatus = TextField(null = True, default = "idle")
shapeExecutionTime = TimeField(null = True)
label = TextField(null = True)
direction = JSONField(null = True)
alpha = FloatField(null = True)
r0 = FloatField(null = True)
L = FloatField(null = True)
radius = FloatField(null = True)
filletsEnabled = BooleanField(null = True)
fillets = FloatField(null = True)
volumeCell = FloatField(null = True)
volume = FloatField(null = True)
porosity = FloatField(null = True)
class Meta:
database = __database_proxy__
table_name = "shapes"
#depends_on = Execution
class Mesh(Model):
mesh_id = AutoField()
shape_id = ForeignKeyField(Shape, backref = "shapes", on_delete = "CASCADE")
meshStatus = TextField(null = True, default = "idle")
meshExecutionTime = TimeField(null = True)
elements = IntegerField(null = True)
edges = IntegerField(null = True)
faces = IntegerField(null = True)
volumes = IntegerField(null = True)
tetrahedrons = IntegerField(null = True)
prisms = IntegerField(null = True)
pyramids = IntegerField(null = True)
class Meta:
database = __database_proxy__
table_name = "meshes"
#depends_on = Execution
class FlowOnephase(Model):
flow_id = AutoField()
mesh_id = ForeignKeyField(Mesh, backref = "meshes", on_delete = "CASCADE")
flowStatus = TextField(null = True, default = "idle")
flowExecutionTime = TimeField(null = True)
pressureInlet = FloatField(null = True)
pressureOutlet = FloatField(null = True)
pressureInternal = FloatField(null = True)
velocityInlet = JSONField(null = True)
velocityOutlet = JSONField(null = True)
velocityInternal = JSONField(null = True)
viscosity = FloatField(null = True)
viscosityKinematic = FloatField(null = True)
density = FloatField(null = True)
flowRate = FloatField(null = True)
permeability = FloatField(null = True)
class Meta:
database = __database_proxy__
table_name = "flows"
#depends_on = Execution
__models__ = [
Execution,
Shape,
Mesh,
FlowOnephase
]

View File

@ -0,0 +1,102 @@
# -*- coding: utf-8 -*-
import peewee as pw
from . import utils
# proxy, assign database later
database_proxy = pw.Proxy()
class Execution(pw.Model):
exec_id = pw.AutoField()
date = pw.DateTimeField()
executionTime = pw.TimeField(null = True)
class Meta:
database = database_proxy
table_name = "executions"
class Shape(pw.Model):
shape_id = pw.AutoField()
exec_id = pw.ForeignKeyField(Execution, backref = "executions", on_delete = "CASCADE")
shapeStatus = pw.TextField(null = True, default = "idle")
shapeExecutionTime = pw.TimeField(null = True)
label = pw.TextField(null = True)
direction = utils.JSONField(null = True)
alpha = pw.FloatField(null = True)
r0 = pw.FloatField(null = True)
L = pw.FloatField(null = True)
radius = pw.FloatField(null = True)
filletsEnabled = pw.BooleanField(null = True)
fillets = pw.FloatField(null = True)
volumeCell = pw.FloatField(null = True)
volume = pw.FloatField(null = True)
porosity = pw.FloatField(null = True)
class Meta:
database = database_proxy
table_name = "shapes"
# depends_on = Execution
class Mesh(pw.Model):
mesh_id = pw.AutoField()
shape_id = pw.ForeignKeyField(Shape, backref = "shapes", on_delete = "CASCADE")
meshStatus = pw.TextField(null = True, default = "idle")
meshExecutionTime = pw.TimeField(null = True)
elements = pw.IntegerField(null = True)
edges = pw.IntegerField(null = True)
faces = pw.IntegerField(null = True)
volumes = pw.IntegerField(null = True)
tetrahedrons = pw.IntegerField(null = True)
prisms = pw.IntegerField(null = True)
pyramids = pw.IntegerField(null = True)
class Meta:
database = database_proxy
table_name = "meshes"
# depends_on = Execution
class FlowOnephase(pw.Model):
flow_id = pw.AutoField()
mesh_id = pw.ForeignKeyField(Mesh, backref = "meshes", on_delete = "CASCADE")
flowStatus = pw.TextField(null = True, default = "idle")
flowExecutionTime = pw.TimeField(null = True)
pressureInlet = pw.FloatField(null = True)
pressureOutlet = pw.FloatField(null = True)
pressureInternal = pw.FloatField(null = True)
velocityInlet = utils.JSONField(null = True)
velocityOutlet = utils.JSONField(null = True)
velocityInternal = utils.JSONField(null = True)
viscosity = pw.FloatField(null = True)
viscosityKinematic = pw.FloatField(null = True)
density = pw.FloatField(null = True)
flowRate = pw.FloatField(null = True)
permeability = pw.FloatField(null = True)
class Meta:
database = database_proxy
table_name = "flows"
# depends_on = Execution
__all__ = [
"Execution",
"Shape",
"Mesh",
"FlowOnephase"
]

View File

@ -1,22 +1,12 @@
# -*- coding: utf-8 -*-
# This file is part of anisotropy.
# License: GNU GPL version 3, see the file "LICENSE" for details.
from peewee import (
TextField,
ColumnBase,
Value,
fn,
Node,
Expression,
OP,
Field
)
import json
from numpy import ndarray
import peewee as pw
import json
class ListField(TextField):
class ListField(pw.TextField):
field_type = "list"
def db_value(self, value):
@ -29,88 +19,78 @@ class ListField(TextField):
try:
pval.append(float(entry))
except:
except Exception:
pval.append(entry.strip().replace("'", ""))
return pval
"""
class JSONField(TextField):
# TODO: fix double quotes when use __eq__ in 'where' method
field_type = "TEXT"
def db_value(self, value):
if isinstance(value, ndarray):
formatted = list(value)
else:
formatted = value
return json.dumps(formatted)
def python_value(self, value):
if value is not None:
return json.loads(value)
"""
class JSONPath(ColumnBase):
class JSONPath(pw.ColumnBase):
def __init__(self, field, path = None):
super(JSONPath, self).__init__()
self._field = field
self._path = path or ()
@property
def path(self):
return Value('$%s' % ''.join(self._path))
return pw.Value('$%s' % ''.join(self._path))
def __getitem__(self, idx):
if isinstance(idx, int):
item = '[%s]' % idx
else:
item = '.%s' % idx
return JSONPath(self._field, self._path + (item,))
def set(self, value, as_json = None):
if as_json or isinstance(value, (list, dict)):
value = fn.json(self._field._json_dumps(value))
return fn.json_set(self._field, self.path, value)
value = pw.fn.json(self._field._json_dumps(value))
return pw.fn.json_set(self._field, self.path, value)
def update(self, value):
return self.set(fn.json_patch(self, self._field._json_dumps(value)))
return self.set(pw.fn.json_patch(self, self._field._json_dumps(value)))
def remove(self):
return fn.json_remove(self._field, self.path)
return pw.fn.json_remove(self._field, self.path)
def json_type(self):
return fn.json_type(self._field, self.path)
return pw.fn.json_type(self._field, self.path)
def length(self):
return fn.json_array_length(self._field, self.path)
return pw.fn.json_array_length(self._field, self.path)
def children(self):
return fn.json_each(self._field, self.path)
return pw.fn.json_each(self._field, self.path)
def tree(self):
return fn.json_tree(self._field, self.path)
return pw.fn.json_tree(self._field, self.path)
def __sql__(self, ctx):
return ctx.sql(fn.json_extract(self._field, self.path)
if self._path else self._field)
return ctx.sql(
pw.fn.json_extract(self._field, self.path)
if self._path else self._field
)
class JSONField(TextField):
class JSONField(pw.TextField):
field_type = 'TEXT'
unpack = False
def __init__(self, json_dumps = None, json_loads = None, **kwargs):
super(JSONField, self).__init__(**kwargs)
self._json_dumps = json_dumps or json.dumps
self._json_loads = json_loads or json.loads
super(JSONField, self).__init__(**kwargs)
def python_value(self, value):
if value is not None:
try:
return json.loads(value)
except (TypeError, ValueError):
return value
@ -119,23 +99,27 @@ class JSONField(TextField):
if isinstance(value, ndarray):
value = list(value)
if not isinstance(value, Node):
if not isinstance(value, pw.Node):
value = json.dumps(value)
return value
def _e(op):
def inner(self, rhs):
if isinstance(rhs, (list, dict)):
rhs = Value(rhs, converter=self.db_value, unpack=False)
return Expression(self, op, rhs)
rhs = pw.Value(rhs, converter = self.db_value, unpack = False)
return pw.Expression(self, op, rhs)
return inner
__eq__ = _e(OP.EQ)
__ne__ = _e(OP.NE)
__gt__ = _e(OP.GT)
__ge__ = _e(OP.GTE)
__lt__ = _e(OP.LT)
__le__ = _e(OP.LTE)
__hash__ = Field.__hash__
__eq__ = _e(pw.OP.EQ)
__ne__ = _e(pw.OP.NE)
__gt__ = _e(pw.OP.GT)
__ge__ = _e(pw.OP.GTE)
__lt__ = _e(pw.OP.LT)
__le__ = _e(pw.OP.LTE)
__hash__ = pw.Field.__hash__
def __getitem__(self, item):
return JSONPath(self)[item]
@ -150,10 +134,10 @@ class JSONField(TextField):
return JSONPath(self).remove()
def json_type(self):
return fn.json_type(self)
return pw.fn.json_type(self)
def length(self):
return fn.json_array_length(self)
return pw.fn.json_array_length(self)
def children(self):
"""
@ -170,8 +154,7 @@ class JSONField(TextField):
json JSON hidden (1st input parameter to function)
root TEXT hidden (2nd input parameter, path at which to start)
"""
return fn.json_each(self)
return pw.fn.json_each(self)
def tree(self):
return fn.json_tree(self)
return pw.fn.json_tree(self)