directory, file, repository, tests

This commit is contained in:
L-Nafaryus 2024-08-05 01:28:20 +05:00
parent 69a1aa2471
commit 383d7c57ab
Signed by: L-Nafaryus
GPG Key ID: 553C97999B363D38
6 changed files with 497 additions and 204 deletions

View File

@ -5,7 +5,29 @@
groups = ["default", "dev"] groups = ["default", "dev"]
strategy = ["cross_platform", "inherit_metadata"] strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1" lock_version = "4.4.1"
content_hash = "sha256:1b3ad8e836b4c01729baf2a537f2c7c543495cd762f914bb7cfd518b1634ab2d" content_hash = "sha256:fe3214096aaef3097e2009f717762fb370bb726aa89a52e7b2a40d60016be987"
[[package]]
name = "aiofiles"
version = "24.1.0"
requires_python = ">=3.8"
summary = "File support for asyncio."
groups = ["default"]
files = [
{file = "aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5"},
{file = "aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c"},
]
[[package]]
name = "aioshutil"
version = "1.5"
requires_python = ">=3.8"
summary = "Asynchronous shutil module."
groups = ["default"]
files = [
{file = "aioshutil-1.5-py3-none-any.whl", hash = "sha256:bc2a6cdcf1a8615b62f856154fd81131031d03f2834912ebb06d8a2391253652"},
{file = "aioshutil-1.5.tar.gz", hash = "sha256:2756d6cd3bb03405dc7348ac11a0b60eb949ebd63cdd15f56e922410231c1201"},
]
[[package]] [[package]]
name = "aiosmtplib" name = "aiosmtplib"

View File

@ -34,6 +34,8 @@ dependencies = [
"cryptography>=43.0.0", "cryptography>=43.0.0",
"python-multipart>=0.0.9", "python-multipart>=0.0.9",
"jinja2>=3.1.4", "jinja2>=3.1.4",
"aiofiles>=24.1.0",
"aioshutil>=1.5",
] ]
requires-python = ">=3.12,<3.13" requires-python = ">=3.12,<3.13"
readme = "README.md" readme = "README.md"

View File

@ -2,8 +2,9 @@ from time import time
from typing import List, Optional, Self from typing import List, Optional, Self
from pathlib import Path from pathlib import Path
import shutil import shutil
import aiofiles
from sqlalchemy import BigInteger, ForeignKey from sqlalchemy import BigInteger, ForeignKey, inspect
from sqlalchemy.orm import mapped_column, Mapped, relationship from sqlalchemy.orm import mapped_column, Mapped, relationship
import sqlalchemy as sa import sqlalchemy as sa
from pydantic import BaseModel, ConfigDict from pydantic import BaseModel, ConfigDict
@ -11,6 +12,7 @@ from pydantic import BaseModel, ConfigDict
from materia.models.base import Base from materia.models.base import Base
from materia.models import database from materia.models import database
from materia.models.database import SessionContext from materia.models.database import SessionContext
from materia.config import Config
class DirectoryError(Exception): class DirectoryError(Exception):
@ -33,193 +35,68 @@ class Directory(Base):
is_public: Mapped[bool] = mapped_column(default=False) is_public: Mapped[bool] = mapped_column(default=False)
repository: Mapped["Repository"] = relationship(back_populates="directories") repository: Mapped["Repository"] = relationship(back_populates="directories")
directories: Mapped[List["Directory"]] = relationship( directories: Mapped[List["Directory"]] = relationship(back_populates="parent")
back_populates="parent", remote_side=[id] parent: Mapped["Directory"] = relationship(
back_populates="directories", remote_side=[id]
) )
parent: Mapped["Directory"] = relationship(back_populates="directories")
files: Mapped[List["File"]] = relationship(back_populates="parent") files: Mapped[List["File"]] = relationship(back_populates="parent")
link: Mapped["DirectoryLink"] = relationship(back_populates="directory") link: Mapped["DirectoryLink"] = relationship(back_populates="directory")
async def new( async def new(self, session: SessionContext, config: Config) -> Optional[Self]:
self,
session: SessionContext,
path: Optional[Path] = None,
with_parents: bool = False,
) -> Optional[Self]:
session.add(self) session.add(self)
await session.flush() await session.flush()
await session.refresh(self, attribute_names=["repository", "parent"]) await session.refresh(self, attribute_names=["repository"])
repository_path: Path = await self.repository.path(session) relative_path = await self.relative_path(session)
current_path: Path = repository_path directory_path = await self.path(session, config)
current_directory: Optional[Directory] = None
for part in path.parts:
current_path /= part
relative_path = current_path.relative_to(repository_path)
if current_path.exists() and current_path.is_dir():
# Find record
current_directory = await Directory.find(
self.repository, self.parent, self.name, session
)
if not current_directory:
# TODO: recreate record
raise DirectoryError(
f"No directory was found in the records: {relative_path}"
)
current_directory.updated = time()
await session.flush()
continue
if not with_parents:
raise DirectoryError(f"Directory not exists at /{relative_path}")
# Create an ancestor directory from scratch
current_directory = await Directory(
repository_id=self.repository.id,
parent_id=current_directory.id if current_directory else None,
name=part,
).new(
session,
path=relative_path,
with_parents=False,
)
try: try:
current_path.mkdir() directory_path.mkdir()
except OSError as e: except OSError as e:
raise DirectoryError( raise DirectoryError(
f"Failed to create directory at /{relative_path}:", *e.args f"Failed to create directory at /{relative_path}:",
*e.args,
) )
# Create directory
current_path /= self.name
relative_path = current_path.relative_to(repository_path)
try:
current_path.mkdir()
except OSError as e:
raise DirectoryError(
f"Failed to create directory at /{relative_path}:", *e.args
)
# Update information
self.parent = current_directory
await session.commit()
return self return self
async def remove(self, session: SessionContext): async def remove(self, session: SessionContext, config: Config):
session.add(self) session.add(self)
await session.refresh(self, attribute_names=["directories", "files"])
current_path: Path = self.repository.path(session) / self.path(session) if self.directories:
for directory in self.directories:
directory.remove(session, config)
if self.files:
for file in self.files:
file.remove(session, config)
relative_path = await self.relative_path(session)
directory_path = await self.path(session, config)
try: try:
shutil.tmtree(str(current_path)) shutil.rmtree(str(directory_path))
except OSError as e: except OSError as e:
raise DirectoryError("Failed to remove directory:", *e.args) raise DirectoryError(
f"Failed to remove directory at /{relative_path}:", *e.args
await session.refresh(self, attribute_names=["parent"]) )
current_directory: Directory = self.parent
while current_directory:
current_directory.updated = time()
session.add(current_directory)
await session.refresh(self, attribute_names=["parent"])
current_directory = current_directory.parent
await session.delete(self) await session.delete(self)
await session.commit()
async def is_root(self) -> bool:
return self.parent_id is None
@staticmethod
async def find(
repository: "Repository",
directory: "Directory",
name: str,
session: SessionContext,
) -> Optional[Self]:
return (
await session.scalars(
sa.select(Directory).where(
sa.and_(
Directory.repository_id == repository.id,
Directory.name == name,
Directory.parent_id == directory.parent_id,
)
)
)
).first()
async def find_nested(self, session: SessionContext):
pass
async def find_by_descend(
self, path: Path | str, db: database.Database, need_create: bool = False
) -> Optional[Self]:
"""Find a nested directory from current"""
repository_id = self.repository_id
path = Path(path)
current_directory = self
async with db.session() as session:
for part in path.parts:
directory = (
await session.scalars(
sa.select(Directory).where(
sa.and_(
Directory.repository_id == repository_id,
Directory.name == part,
Directory.parent_id == current_directory.id,
)
)
)
).first()
if directory is None:
if not need_create:
return None
directory = Directory(
repository_id=repository_id,
parent_id=current_directory.id,
name=part,
)
session.add(directory)
await session.flush() await session.flush()
current_directory = directory async def relative_path(self, session: SessionContext) -> Optional[Path]:
if need_create:
await session.commit()
return current_directory
@staticmethod
async def find_by_path(
repository_id: int, path: Path | str, db: database.Database
) -> Optional[Self]:
"""Find a directory by given absolute path"""
path = Path(path)
assert path == Path(), "The path cannot be empty"
root = await Directory.find_by_descend(repository_id, path.parts[0], db)
return root.descend(Path().joinpath(*path.parts[1:]), db)
async def path(self, session: SessionContext) -> Optional[Path]:
"""Get relative path of the current directory""" """Get relative path of the current directory"""
if inspect(self).was_deleted:
return None
parts = [] parts = []
current_directory = self current_directory = self
async with session.begin_nested():
while True: while True:
parts.append(current_directory.name) parts.append(current_directory.name)
session.add(current_directory) session.add(current_directory)
await session.refresh(current_directory, attribute_names=["parent"]) await session.refresh(current_directory, attribute_names=["parent"])
@ -230,6 +107,87 @@ class Directory(Base):
return Path().joinpath(*reversed(parts)) return Path().joinpath(*reversed(parts))
async def path(self, session: SessionContext, config: Config) -> Optional[Path]:
if inspect(self).was_deleted:
return None
repository_path = await self.repository.path(session, config)
relative_path = await self.relative_path(session)
return repository_path.joinpath(relative_path)
def is_root(self) -> bool:
return self.parent_id is None
@staticmethod
async def by_path(
repository: "Repository", path: Path, session: SessionContext, config: Config
) -> Optional[Self]:
if path == Path():
raise DirectoryError("Cannot find directory by empty path")
current_directory: Optional[Directory] = None
for part in path.parts:
current_directory = (
await session.scalars(
sa.select(Directory).where(
sa.and_(
Directory.repository_id == repository.id,
Directory.name == part,
(
Directory.parent_id == current_directory.id
if current_directory
else Directory.parent_id.is_(None)
),
)
)
)
).first()
if not current_directory:
return None
return current_directory
async def copy(
self, directory: Optional["Directory"], session: SessionContext, config: Config
) -> Self:
pass
async def move(
self, directory: Optional["Directory"], session: SessionContext, config: Config
) -> Self:
pass
async def rename(self, name: str, session: SessionContext, config: Config) -> Self:
session.add(self)
directory_path = await self.path(session, config)
relative_path = await self.relative_path(session)
new_path = directory_path.with_name(name)
identity = 1
while True:
if new_path == directory_path:
break
if not new_path.exists():
break
new_path = directory_path.with_name(f"{name}.{str(identity)}")
identity += 1
try:
await aiofiles.os.rename(directory_path, new_path)
except OSError as e:
raise DirectoryError(
f"Failed to rename directory at /{relative_path}", *e.args
)
self.name = new_path.name
await session.flush()
return self
async def info(self) -> "DirectoryInfo": async def info(self) -> "DirectoryInfo":
return DirectoryInfo.model_validate(self) return DirectoryInfo.model_validate(self)

View File

@ -1,14 +1,23 @@
from time import time from time import time
from typing import Optional, Self from typing import Optional, Self
from pathlib import Path from pathlib import Path
import aioshutil
from sqlalchemy import BigInteger, ForeignKey from sqlalchemy import BigInteger, ForeignKey, inspect
from sqlalchemy.orm import mapped_column, Mapped, relationship from sqlalchemy.orm import mapped_column, Mapped, relationship
import sqlalchemy as sa import sqlalchemy as sa
from pydantic import BaseModel, ConfigDict from pydantic import BaseModel, ConfigDict
import aiofiles
import aiofiles.os
from materia.models.base import Base from materia.models.base import Base
from materia.models import database from materia.models import database
from materia.models.database import SessionContext
from materia.config import Config
class FileError(Exception):
pass
class File(Base): class File(Base):
@ -24,40 +33,184 @@ class File(Base):
created: Mapped[int] = mapped_column(BigInteger, nullable=False, default=time) created: Mapped[int] = mapped_column(BigInteger, nullable=False, default=time)
updated: Mapped[int] = mapped_column(BigInteger, nullable=False, default=time) updated: Mapped[int] = mapped_column(BigInteger, nullable=False, default=time)
name: Mapped[str] name: Mapped[str]
path: Mapped[str] = mapped_column(nullable=True)
is_public: Mapped[bool] = mapped_column(default=False) is_public: Mapped[bool] = mapped_column(default=False)
size: Mapped[int] = mapped_column(BigInteger) size: Mapped[int] = mapped_column(BigInteger, nullable=True)
repository: Mapped["Repository"] = relationship(back_populates="files") repository: Mapped["Repository"] = relationship(back_populates="files")
parent: Mapped["Directory"] = relationship(back_populates="files") parent: Mapped["Directory"] = relationship(back_populates="files")
link: Mapped["FileLink"] = relationship(back_populates="file") link: Mapped["FileLink"] = relationship(back_populates="file")
async def new(
self, data: bytes, session: SessionContext, config: Config
) -> Optional[Self]:
session.add(self)
await session.flush()
await session.refresh(self, attribute_names=["repository"])
relative_path = await self.relative_path(session)
file_path = await self.path(session, config)
size = None
try:
async with aiofiles.open(file_path, mode="wb") as file:
await file.write(data)
size = (await aiofiles.os.stat(file_path)).st_size
except OSError as e:
raise FileError(f"Failed to write file at /{relative_path}", *e.args)
self.size = size
await session.flush()
return self
async def remove(self, session: SessionContext, config: Config):
session.add(self)
relative_path = await self.relative_path(session)
file_path = await self.path(session, config)
try:
await aiofiles.os.remove(file_path)
except OSError as e:
raise FileError(f"Failed to remove file at /{relative_path}:", *e.args)
await session.delete(self)
await session.flush()
async def relative_path(self, session: SessionContext) -> Optional[Path]:
if inspect(self).was_deleted:
return None
file_path = Path()
async with session.begin_nested():
session.add(self)
await session.refresh(self, attribute_names=["parent"])
if self.parent:
file_path = await self.parent.relative_path(session)
return file_path.joinpath(self.name)
async def path(self, session: SessionContext, config: Config) -> Optional[Path]:
if inspect(self).was_deleted:
return None
file_path = Path()
async with session.begin_nested():
session.add(self)
await session.refresh(self, attribute_names=["repository", "parent"])
if self.parent:
file_path = await self.parent.path(session, config)
else:
file_path = await self.repository.path(session, config)
return file_path.joinpath(self.name)
@staticmethod @staticmethod
async def by_path( async def by_path(
repository_id: int, path: Path | None, name: str, db: database.Database repository: "Repository", path: Path, session: SessionContext, config: Config
) -> Self | None: ) -> Optional[Self]:
async with db.session() as session: if path == Path():
query_path = ( raise FileError("Cannot find file by empty path")
File.path == str(path)
if isinstance(path, Path) parent_directory = await Directory.by_path(
else File.path.is_(None) repository, path.parent, session, config
) )
return (
current_file = (
await session.scalars( await session.scalars(
sa.select(File).where( sa.select(File).where(
sa.and_( sa.and_(
File.repository_id == repository_id, File.repository_id == repository.id,
File.name == name, File.name == path.name,
query_path, (
File.parent_id == parent_directory.id
if parent_directory
else File.parent_id.is_(None)
),
) )
) )
) )
).first() ).first()
async def remove(self, db: database.Database): return current_file
async with db.session() as session:
await session.delete(self) async def copy(
await session.commit() self, directory: Optional["Directory"], session: SessionContext, config: Config
) -> Self:
pass
async def move(
self, directory: Optional["Directory"], session: SessionContext, config: Config
) -> Self:
session.add(self)
await session.refresh(self, attribute_names=["repository"])
repository_path = await self.repository.path(session, config)
file_path = await self.path(session, config)
directory_path = (
await directory.path(session, config) if directory else repository_path
)
new_path = File.generate_name(file_path, directory_path, self.name)
try:
await aioshutil.move(file_path, new_path)
except OSError as e:
raise FileError("Failed to move file:", *e.args)
self.parent_id = directory.id if directory else None
await session.flush()
return self
@staticmethod
def generate_name(old_file: Path, target_directory: Path, name: str) -> Path:
new_path = target_directory.joinpath(name)
identity = 1
while True:
if new_path == old_file:
break
if not new_path.exists():
break
new_path = target_directory.joinpath(
f"{name.removesuffix(new_path.suffix)}.{str(identity)}{new_path.suffix}"
)
identity += 1
return new_path
async def rename(self, name: str, session: SessionContext, config: Config) -> Self:
session.add(self)
file_path = await self.path(session, config)
relative_path = await self.relative_path(session)
new_path = File.generate_name(file_path, file_path.parent, name)
try:
await aiofiles.os.rename(file_path, new_path)
except OSError as e:
raise FileError(f"Failed to rename file at /{relative_path}", *e.args)
self.name = new_path.name
await session.flush()
return self
async def info(self) -> Optional["FileInfo"]:
if self.is_public:
return FileInfo.model_validate(self)
return None
def convert_bytes(size: int):
for unit in ["bytes", "kB", "MB", "GB", "TB"]:
if size < 1024:
return f"{size}{unit}" if unit == "bytes" else f"{size:.1f}{unit}"
size >>= 10
class FileLink(Base): class FileLink(Base):
@ -80,7 +233,6 @@ class FileInfo(BaseModel):
created: int created: int
updated: int updated: int
name: str name: str
path: Optional[str]
is_public: bool is_public: bool
size: int size: int

View File

@ -34,13 +34,17 @@ class Repository(Base):
async def new(self, session: SessionContext, config: Config) -> Optional[Self]: async def new(self, session: SessionContext, config: Config) -> Optional[Self]:
session.add(self) session.add(self)
await session.flush() await session.flush()
repository_path = await self.path(session, config) repository_path = await self.path(session, config)
relative_path = repository_path.relative_to(
config.application.working_directory
)
try: try:
repository_path.mkdir(parents=True, exist_ok=True) repository_path.mkdir(parents=True, exist_ok=True)
except OSError as e: except OSError as e:
raise RepositoryError( raise RepositoryError(
f"Failed to create repository at /{repository_path.relative_to(config.application.working_directory)}:", f"Failed to create repository at /{relative_path}:",
*e.args, *e.args,
) )

View File

@ -1,6 +1,7 @@
import pytest_asyncio import pytest_asyncio
import pytest import pytest
import os import os
import sys
from pathlib import Path from pathlib import Path
from materia.config import Config from materia.config import Config
from materia.models import ( from materia.models import (
@ -10,6 +11,7 @@ from materia.models import (
Repository, Repository,
Directory, Directory,
RepositoryError, RepositoryError,
File,
) )
from materia.models.base import Base from materia.models.base import Base
from materia.models.database import SessionContext from materia.models.database import SessionContext
@ -17,7 +19,9 @@ from materia import security
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.pool import NullPool from sqlalchemy.pool import NullPool
from sqlalchemy.orm.session import make_transient from sqlalchemy.orm.session import make_transient
from dataclasses import dataclass from sqlalchemy import inspect
import aiofiles
import aiofiles.os
@pytest_asyncio.fixture(scope="session") @pytest_asyncio.fixture(scope="session")
@ -188,17 +192,19 @@ async def test_repository(data, tmpdir, session: SessionContext, config: Config)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_directory(data, tmpdir, session: SessionContext, config: Config): async def test_directory(data, tmpdir, session: SessionContext, config: Config):
config.application.working_directory = Path(tmpdir)
# setup # setup
session.add(data.user) session.add(data.user)
await session.flush() await session.flush()
repository = Repository(user_id=data.user.id, capacity=config.repository.capacity) repository = await Repository(
session.add(repository) user_id=data.user.id, capacity=config.repository.capacity
await session.flush() ).new(session, config)
directory = Directory(repository_id=repository.id, parent_id=None, name="test1") directory = await Directory(
session.add(directory) repository_id=repository.id, parent_id=None, name="test1"
await session.flush() ).new(session, config)
# simple # simple
assert directory.id is not None assert directory.id is not None
@ -212,15 +218,14 @@ async def test_directory(data, tmpdir, session: SessionContext, config: Config):
) )
) )
).first() == directory ).first() == directory
assert (await directory.path(session, config)).exists()
# nested simple # nested simple
nested_directory = Directory( nested_directory = await Directory(
repository_id=repository.id, repository_id=repository.id,
parent_id=directory.id, parent_id=directory.id,
name="test_nested", name="test_nested",
) ).new(session, config)
session.add(nested_directory)
await session.flush()
assert nested_directory.id is not None assert nested_directory.id is not None
assert ( assert (
@ -234,3 +239,153 @@ async def test_directory(data, tmpdir, session: SessionContext, config: Config):
) )
).first() == nested_directory ).first() == nested_directory
assert nested_directory.parent_id == directory.id assert nested_directory.parent_id == directory.id
assert (await nested_directory.path(session, config)).exists()
# relationship
await session.refresh(directory, attribute_names=["directories", "files"])
assert isinstance(directory.files, list) and len(directory.files) == 0
assert isinstance(directory.directories, list) and len(directory.directories) == 1
await session.refresh(nested_directory, attribute_names=["directories", "files"])
assert (nested_directory.files, list) and len(nested_directory.files) == 0
assert (nested_directory.directories, list) and len(
nested_directory.directories
) == 0
#
assert (
await Directory.by_path(
repository, Path("test1", "test_nested"), session, config
)
== nested_directory
)
# remove nested
nested_path = await nested_directory.path(session, config)
assert nested_path.exists()
await nested_directory.remove(session, config)
assert inspect(nested_directory).was_deleted
assert await nested_directory.path(session, config) is None
assert not nested_path.exists()
await session.refresh(directory) # update attributes that was deleted
assert (await directory.path(session, config)).exists()
# rename
assert (await directory.rename("test1", session, config)).name == "test1"
directory2 = await Directory(
repository_id=repository.id, parent_id=None, name="test2"
).new(session, config)
assert (await directory.rename("test2", session, config)).name == "test2.1"
assert (await repository.path(session, config)).joinpath("test2.1").exists()
assert not (await repository.path(session, config)).joinpath("test1").exists()
directory_path = await directory.path(session, config)
assert directory_path.exists()
await directory.remove(session, config)
assert await directory.path(session, config) is None
assert not directory_path.exists()
@pytest.mark.asyncio
async def test_file(data, tmpdir, session: SessionContext, config: Config):
config.application.working_directory = Path(tmpdir)
# setup
session.add(data.user)
await session.flush()
repository = await Repository(
user_id=data.user.id, capacity=config.repository.capacity
).new(session, config)
directory = await Directory(
repository_id=repository.id, parent_id=None, name="test1"
).new(session, config)
directory2 = await Directory(
repository_id=repository.id, parent_id=None, name="test2"
).new(session, config)
data = b"Hello there, it's a test"
file = await File(
repository_id=repository.id,
parent_id=directory.id,
name="test_file.txt",
).new(data, session, config)
# simple
assert file.id is not None
assert (
await session.scalars(
sa.select(File).where(
sa.and_(
File.repository_id == repository.id,
File.parent_id == directory.id,
File.name == "test_file.txt",
)
)
)
).first() == file
# relationship
await session.refresh(file, attribute_names=["parent", "repository"])
assert file.parent == directory
assert file.repository == repository
#
assert (
await File.by_path(repository, Path("test1", "test_file.txt"), session, config)
== file
)
#
file_path = await file.path(session, config)
assert file_path.exists()
assert (await aiofiles.os.stat(file_path)).st_size == file.size
async with aiofiles.open(file_path, mode="rb") as io:
content = await io.read()
assert data == content
# rename
assert (
await file.rename("test_file_rename.txt", session, config)
).name == "test_file_rename.txt"
file2 = await File(
repository_id=repository.id, parent_id=directory.id, name="test_file_2.txt"
).new(b"", session, config)
assert (
await file.rename("test_file_2.txt", session, config)
).name == "test_file_2.1.txt"
assert (
(await repository.path(session, config))
.joinpath("test1", "test_file_2.1.txt")
.exists()
)
assert (
not (await repository.path(session, config))
.joinpath("test1", "test_file_rename.txt")
.exists()
)
# move
await file.move(directory2, session, config)
await session.refresh(file, attribute_names=["parent"])
assert file.parent == directory2
assert (
not (await repository.path(session, config))
.joinpath("test1", "test_file_2.1.txt")
.exists()
)
assert (
(await repository.path(session, config))
.joinpath("test2", "test_file_2.1.txt")
.exists()
)
# remove
await file.remove(session, config)
assert not await File.by_path(
repository, Path("test1", "test_file.txt"), session, config
)
assert not file_path.exists()