From e67fcc221682ebf8fbb4e308983c7d9ff3ea8cc0 Mon Sep 17 00:00:00 2001 From: L-Nafaryus Date: Fri, 17 May 2024 01:16:30 +0500 Subject: [PATCH] new: filesystem hierarchy, file upload --- materia/api/__init__.py | 3 +- materia/api/filesystem.py | 86 +++++++++++++++++++ materia/api/middleware.py | 13 +-- materia/api/schema/__init__.py | 1 + materia/api/schema/token.py | 10 +++ materia/api/token.py | 1 + materia/api/user.py | 4 +- materia/db/__init__.py | 4 +- materia/db/directory.py | 29 +++++++ materia/db/file.py | 27 ++++++ materia/db/fs_entity.py | 23 ----- materia/db/link.py | 10 ++- ...on.py => aa2ad59fa7d3_initial_creation.py} | 52 ++++++++--- materia/db/repository.py | 25 ++++++ materia/db/user.py | 6 +- 15 files changed, 247 insertions(+), 47 deletions(-) create mode 100644 materia/api/filesystem.py create mode 100644 materia/api/schema/token.py create mode 100644 materia/db/directory.py create mode 100644 materia/db/file.py delete mode 100644 materia/db/fs_entity.py rename materia/db/migrations/versions/{047a8383cdf1_initial_creation.py => aa2ad59fa7d3_initial_creation.py} (50%) create mode 100644 materia/db/repository.py diff --git a/materia/api/__init__.py b/materia/api/__init__.py index 7b23578..c47fa9e 100644 --- a/materia/api/__init__.py +++ b/materia/api/__init__.py @@ -1,9 +1,10 @@ from fastapi import APIRouter -from materia.api import user +from materia.api import user, filesystem def routes() -> APIRouter: router = APIRouter(prefix = "/api") router.include_router(user.router) + router.include_router(filesystem.router) return router diff --git a/materia/api/filesystem.py b/materia/api/filesystem.py new file mode 100644 index 0000000..c09d61b --- /dev/null +++ b/materia/api/filesystem.py @@ -0,0 +1,86 @@ +import os +import time +from pathlib import Path +from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, status +from sqlalchemy import and_, insert, select, update + +from materia import db +from materia.api.depends import DatabaseState +from materia.api.middleware import JwtMiddleware +from materia.config import Config + + +router = APIRouter(tags = ["filesystem"]) + + +@router.put("/file/upload", dependencies = [Depends(JwtMiddleware())]) +async def upload(request: Request, file: UploadFile, database: DatabaseState = Depends(), directory_path: Path = Path()): + print("hi") + user = request.state.user + repository_path = Config.data_dir() / "repository" / user.login_name.lower() + blacklist = [os.sep, ".", "..", "*"] + directory_path = Path(os.sep.join(filter(lambda part: part not in blacklist, directory_path.parts))) + + async with database.session() as session: + session.add(user) + await session.refresh(user, attribute_names = ["repository"]) + if not (repository := user.repository): + repository = db.Repository( + owner_id = user.id, + capacity = 10 * 1024 * 1024 * 1024 + ) + session.add(repository) + + try: + repository_path.mkdir(parents = True, exist_ok = True) + except OSError: + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Failed to created a repository") + + await session.commit() + + async with database.session() as session: + current_directory = None + current_path = Path() + directory = None + + for part in directory_path.parts: + if not (directory := (await session.scalars(select(db.Directory).where(and_(db.Directory.name == part, db.Directory.path == str(current_path)))))).first(): + directory = db.Directory( + repository_id = repository.id, + parent_id = current_directory.id if current_directory else None, + name = part, + path = str(current_path) + ) + session.add(directory) + + current_directory = directory + current_path /= part + + try: + (repository_path / directory_path).mkdir(parents = True, exist_ok = True) + except OSError: + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Failed to created a directory") + + await session.commit() + + async with database.session() as session: + if file_ := (await session.scalars(select(db.File).where(and_(db.File.name == file.filename, db.File.path == str(directory_path))))).first(): + print(file_.__dict__) + await session.execute(update(db.File).where(db.File.id == file_.id).values(updated_unix = time.time(), size = file.size)) + else: + file_ = db.File( + repository_id = repository.id, + parent_id = directory.id if directory else None, + name = file.filename, + path = str(directory_path), + size = file.size + ) + session.add(file_) + + try: + (repository_path / directory_path / file.filename).write_bytes(await file.read()) + except OSError: + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Failed to write a file") + + await session.commit() + diff --git a/materia/api/middleware.py b/materia/api/middleware.py index 985e1ea..c6861f8 100644 --- a/materia/api/middleware.py +++ b/materia/api/middleware.py @@ -6,22 +6,25 @@ from sqlalchemy import select from pydantic import BaseModel from enum import StrEnum from http import HTTPMethod as HttpMethod +from fastapi.security import HTTPBearer from materia.api.depends import ConfigState, DatabaseState from materia.api.token import TokenClaims from materia import db -class JwtMiddleware: - def __init__(self): +class JwtMiddleware(HTTPBearer): + def __init__(self, auto_error: bool = True): + super().__init__(auto_error = auto_error) self.claims: Optional[TokenClaims] = None async def __call__(self, request: Request, config: ConfigState = Depends(), database: DatabaseState = Depends()): if token := request.cookies.get("token"): pass - elif token := request.headers.get("Authorization"): - token = token.strip("Bearer ") - else: + elif (credentials := await super().__call__(request)) and credentials.scheme == "Bearer": + token = credentials.credentials + + if not token: raise HTTPException(status.HTTP_400_BAD_REQUEST, "Missing token") try: diff --git a/materia/api/schema/__init__.py b/materia/api/schema/__init__.py index bee21fd..35e1080 100644 --- a/materia/api/schema/__init__.py +++ b/materia/api/schema/__init__.py @@ -1 +1,2 @@ from materia.api.schema.user import NewUser, User, RemoveUser, LoginUser +from materia.api.schema.token import Token diff --git a/materia/api/schema/token.py b/materia/api/schema/token.py new file mode 100644 index 0000000..cf589b6 --- /dev/null +++ b/materia/api/schema/token.py @@ -0,0 +1,10 @@ +from typing import Optional, Self +from uuid import UUID +from pydantic import BaseModel + +from materia.api.token import TokenClaims + + +class Token(BaseModel): + access_token: str + diff --git a/materia/api/token.py b/materia/api/token.py index a0cf615..442dd46 100644 --- a/materia/api/token.py +++ b/materia/api/token.py @@ -24,3 +24,4 @@ class TokenClaims(BaseModel): data = jwt.decode(token, secret, algorithms = ["HS256"]) return TokenClaims(**data) + diff --git a/materia/api/user.py b/materia/api/user.py index 6c43252..b7ac88c 100644 --- a/materia/api/user.py +++ b/materia/api/user.py @@ -54,7 +54,7 @@ async def remove(body: schema.RemoveUser, database: DatabaseState = Depends()): await session.execute(delete(db.User).where(db.User.id == body.id)) await session.commit() -@router.post("/user/login", status_code = 200) +@router.post("/user/login", status_code = 200, response_model = schema.Token) async def login(body: schema.LoginUser, response: Response, database: DatabaseState = Depends(), config: ConfigState = Depends()) -> Any: query = select(db.User) if login := body.login: @@ -86,6 +86,8 @@ async def login(body: schema.LoginUser, response: Response, database: DatabaseSt samesite = "none" ) + return schema.Token(access_token = token) + @router.get("/user/logout", status_code = 200) async def logout(response: Response): response.set_cookie( diff --git a/materia/db/__init__.py b/materia/db/__init__.py index a7fc7c9..b74a329 100644 --- a/materia/db/__init__.py +++ b/materia/db/__init__.py @@ -11,7 +11,9 @@ from asyncpg import Connection from materia.db.base import Base from materia.db.user import User -from materia.db.fs_entity import FsEntity +from materia.db.repository import Repository +from materia.db.directory import Directory +from materia.db.file import File from materia.db.link import Link diff --git a/materia/db/directory.py b/materia/db/directory.py new file mode 100644 index 0000000..464b4f9 --- /dev/null +++ b/materia/db/directory.py @@ -0,0 +1,29 @@ +from time import time +from typing import List + +from sqlalchemy import BigInteger, ForeignKey +from sqlalchemy.orm import mapped_column, Mapped, relationship + +from materia.db.base import Base + + +class Directory(Base): + __tablename__ = "directory" + + id: Mapped[int] = mapped_column(BigInteger, primary_key = True) + repository_id: Mapped[int] = mapped_column(ForeignKey("repository.id", ondelete = "CASCADE")) + parent_id: Mapped[int] = mapped_column(ForeignKey("directory.id"), nullable = True) + created_unix: Mapped[int] = mapped_column(BigInteger, nullable = False, default = time) + updated_unix: Mapped[int] = mapped_column(BigInteger, nullable = False, default = time) + name: Mapped[str] + path: Mapped[str] = mapped_column(nullable = True) + is_public: Mapped[bool] = mapped_column(default = False) + + repository: Mapped["Repository"] = relationship(back_populates = "directories") + directories: Mapped[List["Directory"]] = relationship(back_populates = "parent", remote_side = [id]) + parent: Mapped["Directory"] = relationship(back_populates = "directories") + files: Mapped[List["File"]] = relationship(back_populates = "parent") + + +from materia.db.repository import Repository +from materia.db.file import File diff --git a/materia/db/file.py b/materia/db/file.py new file mode 100644 index 0000000..6e434a1 --- /dev/null +++ b/materia/db/file.py @@ -0,0 +1,27 @@ +from time import time + +from sqlalchemy import BigInteger, ForeignKey +from sqlalchemy.orm import mapped_column, Mapped, relationship + +from materia.db.base import Base + + +class File(Base): + __tablename__ = "file" + + id: Mapped[int] = mapped_column(BigInteger, primary_key = True) + repository_id: Mapped[int] = mapped_column(ForeignKey("repository.id", ondelete = "CASCADE")) + parent_id: Mapped[int] = mapped_column(ForeignKey("directory.id"), nullable = True) + created_unix: Mapped[int] = mapped_column(BigInteger, nullable = False, default = time) + updated_unix: Mapped[int] = mapped_column(BigInteger, nullable = False, default = time) + name: Mapped[str] + path: Mapped[str] = mapped_column(nullable = True) + is_public: Mapped[bool] = mapped_column(default = False) + size: Mapped[int] = mapped_column(BigInteger) + + repository: Mapped["Repository"] = relationship(back_populates = "files") + parent: Mapped["Directory"] = relationship(back_populates = "files") + + +from materia.db.directory import Directory +from materia.db.repository import Repository diff --git a/materia/db/fs_entity.py b/materia/db/fs_entity.py deleted file mode 100644 index 0638125..0000000 --- a/materia/db/fs_entity.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import List -from uuid import UUID, uuid4 - -from sqlalchemy import BigInteger, ForeignKey -from sqlalchemy.orm import mapped_column, Mapped, relationship - -from materia.db.base import Base - -class FsEntity(Base): - __tablename__ = "fs_entity" - - id = mapped_column(BigInteger, primary_key = True) - owner_id: Mapped[UUID] = mapped_column(ForeignKey("user.id", ondelete = "CASCADE"), default = uuid4) - parent_id: Mapped[int] = mapped_column(BigInteger, nullable = False) - created_unix: Mapped[int] = mapped_column(BigInteger, nullable = False) - updated_unix: Mapped[int] = mapped_column(BigInteger, nullable = False) - name: Mapped[str] - is_private: Mapped[bool] - is_root: Mapped[bool] - is_file: Mapped[bool] - - children: Mapped[List["FsEntity"]] = relationship(back_populates = "parent") - parent: Mapped["FsEntity"] = relationship(back_populates = "children") diff --git a/materia/db/link.py b/materia/db/link.py index 80fe9a1..910547a 100644 --- a/materia/db/link.py +++ b/materia/db/link.py @@ -8,6 +8,12 @@ class Link(Base): __tablename__ = "link" id: Mapped[int] = mapped_column(BigInteger, primary_key = True) - entity_id: Mapped[int] = mapped_column(ForeignKey("fs_entity.id", ondelete = "CASCADE")) + directory_id: Mapped[int] = mapped_column(ForeignKey("directory.id", ondelete = "CASCADE"), nullable = True) + file_id: Mapped[int] = mapped_column(ForeignKey("directory.id", ondelete = "CASCADE"), nullable = True) created_unix: Mapped[int] = mapped_column(BigInteger, nullable = False, default = time.time) - is_public: Mapped[bool] + is_file: Mapped[bool] + url: Mapped[str] + + +from materia.db.directory import Directory +from materia.db.file import File diff --git a/materia/db/migrations/versions/047a8383cdf1_initial_creation.py b/materia/db/migrations/versions/aa2ad59fa7d3_initial_creation.py similarity index 50% rename from materia/db/migrations/versions/047a8383cdf1_initial_creation.py rename to materia/db/migrations/versions/aa2ad59fa7d3_initial_creation.py index 3c1ae3b..0e11cc2 100644 --- a/materia/db/migrations/versions/047a8383cdf1_initial_creation.py +++ b/materia/db/migrations/versions/aa2ad59fa7d3_initial_creation.py @@ -1,8 +1,8 @@ """Initial creation -Revision ID: 047a8383cdf1 +Revision ID: aa2ad59fa7d3 Revises: -Create Date: 2024-05-15 22:16:29.684146 +Create Date: 2024-05-17 01:14:50.277521 """ from typing import Sequence, Union @@ -12,7 +12,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision: str = '047a8383cdf1' +revision: str = 'aa2ad59fa7d3' down_revision: Union[str, None] = None branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -33,25 +33,49 @@ def upgrade() -> None: sa.Column('created_unix', sa.BigInteger(), nullable=False), sa.PrimaryKeyConstraint('id') ) - op.create_table('fs_entity', + op.create_table('repository', sa.Column('id', sa.BigInteger(), nullable=False), sa.Column('owner_id', sa.Uuid(), nullable=False), - sa.Column('parent_id', sa.BigInteger(), nullable=False), + sa.Column('capacity', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['owner_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('directory', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('repository_id', sa.BigInteger(), nullable=False), + sa.Column('parent_id', sa.BigInteger(), nullable=True), sa.Column('created_unix', sa.BigInteger(), nullable=False), sa.Column('updated_unix', sa.BigInteger(), nullable=False), sa.Column('name', sa.String(), nullable=False), - sa.Column('is_private', sa.Boolean(), nullable=False), - sa.Column('is_root', sa.Boolean(), nullable=False), - sa.Column('is_file', sa.Boolean(), nullable=False), - sa.ForeignKeyConstraint(['owner_id'], ['user.id'], ondelete='CASCADE'), + sa.Column('path', sa.String(), nullable=True), + sa.Column('is_public', sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint(['parent_id'], ['directory.id'], ), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('file', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('repository_id', sa.BigInteger(), nullable=False), + sa.Column('parent_id', sa.BigInteger(), nullable=True), + sa.Column('created_unix', sa.BigInteger(), nullable=False), + sa.Column('updated_unix', sa.BigInteger(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('path', sa.String(), nullable=True), + sa.Column('is_public', sa.Boolean(), nullable=False), + sa.Column('size', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['parent_id'], ['directory.id'], ), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ondelete='CASCADE'), sa.PrimaryKeyConstraint('id') ) op.create_table('link', sa.Column('id', sa.BigInteger(), nullable=False), - sa.Column('entity_id', sa.BigInteger(), nullable=False), + sa.Column('directory_id', sa.BigInteger(), nullable=True), + sa.Column('file_id', sa.BigInteger(), nullable=True), sa.Column('created_unix', sa.BigInteger(), nullable=False), - sa.Column('is_public', sa.Boolean(), nullable=False), - sa.ForeignKeyConstraint(['entity_id'], ['fs_entity.id'], ondelete='CASCADE'), + sa.Column('is_file', sa.Boolean(), nullable=False), + sa.Column('url', sa.String(), nullable=False), + sa.ForeignKeyConstraint(['directory_id'], ['directory.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['file_id'], ['directory.id'], ondelete='CASCADE'), sa.PrimaryKeyConstraint('id') ) # ### end Alembic commands ### @@ -60,6 +84,8 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table('link') - op.drop_table('fs_entity') + op.drop_table('file') + op.drop_table('directory') + op.drop_table('repository') op.drop_table('user') # ### end Alembic commands ### diff --git a/materia/db/repository.py b/materia/db/repository.py new file mode 100644 index 0000000..db4efa5 --- /dev/null +++ b/materia/db/repository.py @@ -0,0 +1,25 @@ +from time import time +from typing import List +from uuid import UUID, uuid4 + +from sqlalchemy import BigInteger, ForeignKey +from sqlalchemy.orm import mapped_column, Mapped, relationship + +from materia.db.base import Base + + +class Repository(Base): + __tablename__ = "repository" + + id: Mapped[int] = mapped_column(BigInteger, primary_key = True) + owner_id: Mapped[UUID] = mapped_column(ForeignKey("user.id")) + capacity: Mapped[int] = mapped_column(BigInteger, nullable = False) + + owner: Mapped["User"] = relationship(back_populates = "repository") + directories: Mapped[List["Directory"]] = relationship(back_populates = "repository") + files: Mapped[List["File"]] = relationship(back_populates = "repository") + + +from materia.db.user import User +from materia.db.directory import Directory +from materia.db.file import File diff --git a/materia/db/user.py b/materia/db/user.py index acd67db..638f33b 100644 --- a/materia/db/user.py +++ b/materia/db/user.py @@ -3,7 +3,7 @@ from typing import Optional import time from sqlalchemy import BigInteger -from sqlalchemy.orm import mapped_column, Mapped +from sqlalchemy.orm import mapped_column, Mapped, relationship from materia.db.base import Base @@ -21,3 +21,7 @@ class User(Base): avatar: Mapped[Optional[str]] created_unix: Mapped[int] = mapped_column(BigInteger, default = time.time) + repository: Mapped["Repository"] = relationship(back_populates = "owner") + + +from materia.db.repository import Repository