Mod: db improved

This commit is contained in:
L-Nafaryus 2021-12-09 13:59:32 +05:00
parent 64a5cf1a6d
commit c35d5cfe3c
6 changed files with 61 additions and 720 deletions

View File

@ -65,8 +65,6 @@ class UltimateRunner(object):
def start(self, queue: list = None, nprocs: int = None):
nprocs = nprocs or self.config["nprocs"]
runners = [ runner.pipeline for runner in self.queue ]
args = [[self.config["stage"]]] * len(self.queue)
parallel = ParallelRunner(nprocs = nprocs)
parallel.start()

View File

@ -1,4 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from .database import *
from .models import __database__, __models__
database = __database__
tables = __models__

View File

@ -3,38 +3,42 @@
# License: GNU GPL version 3, see the file "LICENSE" for details.
import os
from .models import (
sqliteDB,
Execution,
Shape,
Mesh,
FlowOnephase
)
from peewee import SqliteDatabase
class Database(object):
def __init__(self, filename: str):
self.filename = filename
self.database = sqliteDB
def setup(self):
path = os.path.abspath(self.filename)
#os.makedirs(path, exist_ok = True)
class Database(SqliteDatabase):
def __init__(self, *args, **kwargs):
self.filepath = None
self.pragmas_ = kwargs.get("pragmas", { "foreign_keys": 1 })
self.field_types_ = kwargs.get("field_types", { "list": "text" })
self.autoconnect_ = kwargs.get("autoconnect", False)
self.database.init(
path,
pragmas = { "foreign_keys": 1 },
field_types = { "list": "text" },
autoconnect = False
SqliteDatabase.__init__(
self,
None,
pragmas = kwargs.get("pragmas", { "foreign_keys": 1 }),
field_types = kwargs.get("field_types", { "list": "text" }),
autoconnect = kwargs.get("autoconnect", False)
)
if not os.path.exists(path):
with self.database:
self.database.create_tables([Execution])
self.database.create_tables([
Shape,
Mesh,
FlowOnephase
])
@property
def tables(self):
return models.__models__
def setup(self, filename: str):
if not self.filepath:
self.filepath = os.path.abspath(filename) if filename else None
self.init(
self.filepath,
pragmas = self.pragmas_,
#field_types = self.field_types_,
#autoconnect = self.autoconnect_
)
print(self.tables)
self.connect()
self.create_tables(self.tables)
self.close()
# NOTE: avoid circular or partial import
from . import models

View File

@ -11,13 +11,10 @@ from peewee import (
TimeField, DateTimeField
)
from anisotropy.database.utils import JSONField
from .database import Database
sqliteDB = SqliteDatabase(
None,
pragmas = { "foreign_keys": 1 },
field_types = { "list": "text" },
autoconnect = False
)
__database__ = Database()
class Execution(Model):
exec_id = AutoField()
@ -26,7 +23,7 @@ class Execution(Model):
executionTime = TimeField(null = True)
class Meta:
database = sqliteDB
database = __database__
table_name = "executions"
@ -55,9 +52,9 @@ class Shape(Model):
porosityRounded = FloatField(null = True)
class Meta:
database = sqliteDB
database = __database__
table_name = "shapes"
depends_on = Execution
#depends_on = Execution
class Mesh(Model):
@ -77,9 +74,9 @@ class Mesh(Model):
class Meta:
database = sqliteDB
database = __database__
table_name = "meshes"
depends_on = Execution
#depends_on = Execution
class FlowOnephase(Model):
@ -93,8 +90,14 @@ class FlowOnephase(Model):
permeability = FloatField(null = True)
class Meta:
database = sqliteDB
database = __database__
table_name = "flows"
depends_on = Execution
#depends_on = Execution
__models__ = [
Execution,
Shape,
Mesh,
FlowOnephase
]

View File

@ -1,662 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "c4317e7e-d7b9-4bec-9cad-e26a377458b8",
"metadata": {},
"outputs": [],
"source": [
"from peewee import (\n",
" SqliteDatabase, JOIN, \n",
" Model, Field, \n",
" AutoField, ForeignKeyField, \n",
" TextField, FloatField, \n",
" IntegerField, BooleanField, \n",
" TimeField\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "db03ea1b-e950-4f0b-8c4f-c137417981ca",
"metadata": {},
"outputs": [],
"source": [
"db = SqliteDatabase(\"test_db.db\", pragmas = { \"foreign_keys\" : 1, \"journal_mode\": \"wal\" })"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e904a64b-9108-414b-974b-2dbf96018cf5",
"metadata": {},
"outputs": [],
"source": [
"class ListField(Field):\n",
" field_type = \"list\"\n",
"\n",
" def db_value(self, value):\n",
" return str(value)\n",
"\n",
" def python_value(self, value):\n",
" pval = []\n",
"\n",
" for entry in value[1 : -1].split(\",\"):\n",
" try:\n",
" pval.append(float(entry))\n",
"\n",
" except:\n",
" pval.append(entry.strip().replace(\"'\", \"\"))\n",
"\n",
" return pval"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "a6049589-dd33-4cfa-8434-1c00285c01e0",
"metadata": {},
"outputs": [],
"source": [
"class Structure(Model):\n",
" structure_id = AutoField()\n",
"\n",
" type = TextField()\n",
" direction = ListField()\n",
" theta = FloatField()\n",
"\n",
" r0 = FloatField(null = True)\n",
" L = FloatField(null = True)\n",
" radius = FloatField(null = True)\n",
"\n",
" filletsEnabled = BooleanField(null = True)\n",
" fillets = FloatField(null = True)\n",
" #path = TextField()\n",
" \n",
" class Meta:\n",
" database = db\n",
" db_table = \"structures\"\n",
"\n",
"\n",
"class Mesh(Model):\n",
" mesh_id = AutoField()\n",
" structure_id = ForeignKeyField(Structure, backref = \"meshes\")\n",
"\n",
" maxSize = FloatField(null = True) \n",
"\n",
" class Meta:\n",
" database = db\n",
" db_table = \"meshes\"\n",
" depends_on = Structure"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "77c8b505-8b72-4f41-9f6d-d0b4e44884ea",
"metadata": {},
"outputs": [],
"source": [
"db.create_tables([Structure, Mesh])"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "708acdf5-a604-44c1-8631-d274d0273c1b",
"metadata": {},
"outputs": [],
"source": [
"import numpy"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "62c1c0a0-74ae-43cc-bc75-1fbd353540ae",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s1 = Structure(type = \"simple\", direction = [1, 0, 0], theta = 0.01)\n",
"s1.save()\n",
"\n",
"m1 = Mesh(structure_id = s1, maxSize = 1e-2)\n",
"m1.save()\n",
"\n",
"#db.commit()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "0e03bc08-94ea-4680-8223-dbafc13dcfe7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s1.theta = 0.12\n",
"s1.save()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "a2f68e62-0324-4ec2-8113-47cf6e1dc5bd",
"metadata": {},
"outputs": [],
"source": [
"ss = numpy.array([\n",
" Structure(type = \"simple\", direction = [1, 0, 0], theta = 0.01),\n",
" Structure(type = \"simple\", direction = [1, 0, 0], theta = 0.02)\n",
"])"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "6683ea42-5fdc-4b33-b709-5ea31ae5567f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"dtype('O')"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ss.dtype"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "83cc7bde-e003-4f2c-ab9b-f9e37634c374",
"metadata": {},
"outputs": [],
"source": [
"Structure.bulk_create(ss)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "962010a5-68b8-4711-a067-962677aa406d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 simple [1.0, 0.0, 0.0] 0.12\n",
"2 simple [1.0, 0.0, 0.0] 0.01\n",
"3 simple [1.0, 0.0, 0.0] 0.01\n",
"4 simple [1.0, 0.0, 0.0] 0.01\n",
"5 simple [1.0, 0.0, 0.0] 0.01\n",
"6 simple [1.0, 0.0, 0.0] 0.01\n",
"7 simple [1.0, 0.0, 0.0] 0.01\n",
"8 simple [1.0, 0.0, 0.0] 0.01\n",
"9 simple [1.0, 0.0, 0.0] 0.01\n",
"10 simple [1.0, 0.0, 0.0] 0.01\n",
"11 simple [1.0, 0.0, 0.0] 0.01\n",
"12 simple [1.0, 0.0, 0.0] 0.02\n"
]
}
],
"source": [
"for row in Structure.select():\n",
" print(row.structure_id, row.type, row.direction, row.theta)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "a5fc7787-7045-4bb1-a846-5fff8b21417c",
"metadata": {},
"outputs": [],
"source": [
"sel = Structure.select()[-1]\n",
"sel2 = sel.meshes.select()[-1]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "480edddc-478e-47ce-8678-490455badd35",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<Mesh: 8>"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sel2"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "8286c287-f89d-40bd-8045-a854013915c2",
"metadata": {},
"outputs": [],
"source": [
"sel3 = Structure.select().dicts().get()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "180b07cb-6c10-4390-9f9c-07bf9f74f39e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'structure_id': 1,\n",
" 'type': 'simple',\n",
" 'direction': [1.0, 0.0, 0.0],\n",
" 'theta': 0.12,\n",
" 'r0': None,\n",
" 'L': None,\n",
" 'radius': None,\n",
" 'filletsEnabled': None,\n",
" 'fillets': None}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sel3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "448c99bf-2b6f-43de-92d3-dc1267e766d3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 47,
"id": "6f9e3cfd-8945-4738-9bf6-704bca131a9a",
"metadata": {},
"outputs": [],
"source": [
"db = SqliteDatabase(\"newtest2.db\")"
]
},
{
"cell_type": "code",
"execution_count": 48,
"id": "ff82a760-902a-416d-a5d6-775ddb418a9c",
"metadata": {},
"outputs": [],
"source": [
"class Test(Model):\n",
" test_id = AutoField()\n",
" text = TextField(null = True)\n",
" \n",
" class Meta:\n",
" database = db\n",
"\n",
"db.create_tables([Test])"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "a7f89828-9a8d-43ab-92d4-681fd7cbc416",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"False"
]
},
"execution_count": 49,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db.is_closed()"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "3cf907ef-8988-4f08-b1c1-6c3637cefd29",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 50,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test1 = Test(text = \"hola\")\n",
"test1.save()"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "0c91e698-a07b-4b52-bc9d-51a37e179cee",
"metadata": {},
"outputs": [],
"source": [
"test2 = Test.create(text = \"asd\")"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "1deee68c-07b7-4003-a29f-1e726aefc9e0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 hola\n",
"2 asd\n"
]
}
],
"source": [
"for row in Test.select():\n",
" print(row.test_id, row.text)"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "c59d0a44-b907-427a-91d6-d9c9c72c8bdd",
"metadata": {},
"outputs": [],
"source": [
"from multiprocessing import Process, Queue"
]
},
{
"cell_type": "code",
"execution_count": 54,
"id": "88212490-2dd1-4058-80fe-57ae30bf38cc",
"metadata": {},
"outputs": [],
"source": [
"def queue(cmd, qin, qout, *args):\n",
" while True:\n",
" pos, var = qin.get()\n",
" \n",
" if pos is None:\n",
" break\n",
"\n",
" res = cmd(*var, *args)\n",
"\n",
" qout.put((pos, res))\n",
"\n",
" return"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "df6568c8-595d-4c35-9560-16bfe8e3a915",
"metadata": {},
"outputs": [],
"source": [
"def db_save(table):\n",
" with db.atomic():\n",
" return table.save()"
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "53fa9e79-e3e9-4758-a434-4f411dd15858",
"metadata": {},
"outputs": [],
"source": [
"qin = Queue(1)\n",
"qout = Queue()\n",
"procs = []\n",
"nprocs = 10\n",
"\n",
"for n in range(nprocs):\n",
" args = (db_save, qin, qout)\n",
"\n",
" procs.append(Process(target = queue, args = args))"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "86a39b3c-bbb9-4c7e-938b-3330ad7406b9",
"metadata": {},
"outputs": [],
"source": [
"for p in procs:\n",
" p.daemon = True\n",
" p.start()\n",
"\n",
"var = []\n",
"for n in range(50):\n",
" var.append([Test(text = f\"test_{ n }\")])\n",
"\n",
"for n in range(len(var)):\n",
" qin.put((n, var[n]))\n",
"\n",
"for _ in range(nprocs):\n",
" qin.put((None, None))\n",
"\n",
"results = [[] for n in range(len(var))]\n",
"\n",
"for n in range(len(var)):\n",
" index, res = qout.get() \n",
" results[index] = res\n",
"\n",
"for p in procs:\n",
" p.join()"
]
},
{
"cell_type": "code",
"execution_count": 58,
"id": "38e46264-7a61-4215-afe6-827b835ca82b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 58,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"procs[0].exitcode"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "d8da8120-ba3e-44d3-9b9b-78f040c957c0",
"metadata": {},
"outputs": [],
"source": [
"test3 = Test.create(text = \"afterproc\")"
]
},
{
"cell_type": "code",
"execution_count": 60,
"id": "c716656c-7aaa-40da-8582-2311f1dd5314",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 hola\n",
"2 asd\n",
"3 test_1\n",
"4 test_0\n",
"5 test_11\n",
"6 test_12\n",
"7 test_13\n",
"8 test_3\n",
"9 test_9\n",
"10 test_16\n",
"11 test_2\n",
"12 test_7\n",
"13 test_19\n",
"14 test_20\n",
"15 test_4\n",
"16 test_21\n",
"17 test_23\n",
"18 test_24\n",
"19 test_25\n",
"20 test_26\n",
"21 test_27\n",
"22 test_28\n",
"23 test_18\n",
"24 test_30\n",
"25 test_31\n",
"26 test_32\n",
"27 test_14\n",
"28 test_34\n",
"29 test_35\n",
"30 test_6\n",
"31 test_37\n",
"32 test_38\n",
"33 test_39\n",
"34 test_40\n",
"35 test_41\n",
"36 test_36\n",
"37 test_43\n",
"38 test_44\n",
"39 test_42\n",
"40 test_46\n",
"41 test_47\n",
"42 test_48\n",
"43 test_49\n",
"44 test_45\n",
"45 test_29\n",
"46 test_5\n",
"47 test_10\n",
"48 test_15\n",
"49 test_17\n",
"50 test_33\n",
"51 test_22\n",
"52 test_8\n",
"53 afterproc\n"
]
}
],
"source": [
"for row in Test.select():\n",
" print(row.test_id, row.text)"
]
},
{
"cell_type": "code",
"execution_count": 64,
"id": "f46377ad-0af5-4acf-95bb-955e327f51e8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<peewee.SqliteDatabase at 0x7f4cdbc08f70>"
]
},
"execution_count": 64,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "79a02231-365d-4a20-8899-6b5e8dbc8489",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@ -14,26 +14,21 @@ class TestDatabase(unittest.TestCase):
def test_setup(self):
filepath = os.path.join(self.outputPath, "test_database.db")
tables = [
self.database.Execution,
self.database.Physics,
self.database.Shape,
self.database.Mesh,
self.database.Flow
]
db = self.database.Database(filepath)
db.setup()
db = self.database.database
db.setup(filepath)
self.assertTrue(
os.path.exists(filepath) and os.path.isfile(filepath),
"database wasn't created"
)
for table in tables:
self.assertTrue(table.table_exists())
with db:
for table in db.tables:
self.assertTrue(table.table_exists())
def tearDown(self):
pass
os.removedirs(os.outputPath)
if __name__ == "__main__":
unittest.main()