Single commit for HF2

.gitattributes

@@ -0,0 +1,7 @@
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,26 @@
+---
+name: Bug Report
+about: Template for reporting bugs with reproduction steps.
+title: "BUG: [Brief Description]"
+labels: ["bug"]
+assignees: ""
+---
+
+## Current Behavior
+*What actually happens?*
+
+## Expected Behavior
+*What should happen instead?*
+
+## Steps to Reproduce
+1. Go to...
+2. Click on...
+3. Scroll to...
+4. See error
+
+## Environment
+- OS: [e.g., Windows 11]
+- Browser: [e.g., Chrome 120]
+- Version: [e.g., v2.1.0]
+
+## Screenshots/Logs

.github/ISSUE_TEMPLATE/technical_task.md

@@ -0,0 +1,23 @@
+---
+name: Technical Task
+about: Template for technical tasks with subtasks.
+title: "TECH TASK: [Brief Description]"
+labels: ["refactor", "technical"]
+assignees: ""
+---
+
+## Objective
+*What technical goal does this achieve?*
+
+## Subtasks
+- [ ] Subtask 1 (e.g., "Refactor X component")
+- [ ] Subtask 2 (e.g., "Update dependency Y")
+- [ ] Subtask 3 (e.g., "Write tests for Z")
+
+OR
+
+## Linked Subtask Issues
+- #45 (Refactor X)
+- #46 (Update Y)
+
+## Implementation Notes

.github/ISSUE_TEMPLATE/user_story.md

@@ -0,0 +1,16 @@
+---
+name: User Story
+about: Template for capturing user stories with GIVEN/WHEN/THEN acceptance criteria.
+title: "USER STORY: [Brief Description]"
+labels: ["feature"]
+assignees: ""
+---
+
+## Description
+*As a [user role], I want [goal] so that [benefit].*
+
+## Acceptance Criteria (GIVEN/WHEN/THEN)
+```gherkin
+GIVEN [initial context]
+WHEN [action/event occurs]
+THEN [expected outcome]

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,33 @@
+---
+name: Standard Pull Request
+description: Template for all PRs with mandatory issue linking and change documentation.
+title: "PR: [Brief Description]"
+labels: ["needs-review"]
+assignees: ""
+---
+
+## Linked Issue
+<!-- Mandatory: Link to the issue this PR resolves -->
+Fixes #123
+*(or use `Closes #123`, `Resolves #123`)*
+
+## Changes Proposed
+<!-- Describe the changes in this PR -->
+- Added [feature X]
+- Fixed [bug Y]
+- Refactored [component Z]
+
+## Testing Done
+<!-- How was this tested? Include steps or environment details -->
+- [ ] Unit tests
+- [ ] Manual testing (steps: 1. ... 2. ...)
+- Tested on: [OS/Browser]
+
+## Screenshots (if UI changes)
+<!-- Drag & drop images -->
+
+## Checklist
+- [ ] Code follows project style guidelines
+- [ ] Documentation updated (if needed)
+- [ ] Branch is up-to-date with `main`
+- [ ] Reviewer(s) assigned

.github/PULL_REQUEST_TEMPLATE/standart.md

@@ -0,0 +1,33 @@
+---
+name: Standard Pull Request
+description: Template for all PRs with mandatory issue linking and change documentation.
+title: "PR: [Brief Description]"
+labels: ["needs-review"]
+assignees: ""
+---
+
+## Linked Issue
+<!-- Mandatory: Link to the issue this PR resolves -->
+Fixes #123
+*(or use `Closes #123`, `Resolves #123`)*
+
+## Changes Proposed
+<!-- Describe the changes in this PR -->
+- Added [feature X]
+- Fixed [bug Y]
+- Refactored [component Z]
+
+## Testing Done
+<!-- How was this tested? Include steps or environment details -->
+- [ ] Unit tests
+- [ ] Manual testing (steps: 1. ... 2. ...)
+- Tested on: [OS/Browser]
+
+## Screenshots (if UI changes)
+<!-- Drag & drop images -->
+
+## Checklist
+- [ ] Code follows project style guidelines
+- [ ] Documentation updated (if needed)
+- [ ] Branch is up-to-date with `main`
+- [ ] Reviewer(s) assigned

.github/workflows/sync-to-hf.yml

@@ -0,0 +1,99 @@
+name: Sync to Hugging Face Hub
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    environment: Integration test
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+
+      - name: Configure Git identity
+        run: |
+          git config --global user.name "Andrchest"
+          git config --global user.email "andreipolevoi220@gmail.com"
+
+      - name: Set up Git LFS
+        run: |
+          git lfs install
+          git lfs track "*.jpeg" "*.jpg" "*.png" "*.gif" "*.pdf" "*.zip" "*.bin"
+          git add .gitattributes
+          git add .
+          git commit -m "Add LFS tracking for binary files" || true
+
+      - name: Create .env file
+        run: |
+          echo "DATABASE_URL=${{ secrets.DATABASE_URL }}" >> .env
+          echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" >> .env
+          echo "SECRET_PEPPER=${{ secrets.SECRET_PEPPER }}" >> .env
+          echo "JWT_ALGORITHM=${{ secrets.JWT_ALGORITHM }}" >> .env
+          echo "HF1_URL=${{ secrets.HF1_URL }}" >> .env
+
+      - name: Build Docker image
+        run: docker build -t my-app .
+
+      - name: Run Docker container
+        run: docker run -d --name my-app-container -v $(pwd)/.env:/app/.env -p 7860:7860 my-app
+
+      - name: Wait for app to be ready
+        run: |
+          for i in {1..10}; do
+            if curl -s http://localhost:7860/health | grep -q "ok"; then
+              echo "App is ready!"
+              break
+            fi
+            echo "Waiting for app..."
+            sleep 10
+          done
+
+      - name: Install test dependencies inside container
+        run: docker exec my-app-container pip install pytest pytest-cov
+
+      - name: Run integration tests with coverage
+        run: |
+          docker exec -w /app my-app-container python -m pytest app/tests/integration/tests_draft.py -v --cov=app --cov-report=xml --cov-report=html
+
+      - name: Copy coverage reports from container
+        run: |
+          docker cp my-app-container:/app/coverage.xml .
+          docker cp my-app-container:/app/htmlcov .
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-coverage-report
+          path: |
+            coverage.xml
+            htmlcov/
+
+      - name: Remove .env file
+        run: rm .env
+
+      - name: Push to HF2 if tests pass
+        if: success()
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          git lfs install
+          git lfs track "*.jpeg" "*.jpg" "*.png" "*.gif" "*.pdf" "*.zip" "*.bin"
+          git add .gitattributes
+          git add .
+          git commit -m "Add LFS tracking for binary files" || true
+          git checkout -b hf2-single-commit
+          git reset --soft $(git rev-list --max-parents=0 HEAD)
+          git commit -m "Single commit for HF2"
+          git remote add hf2 https://Andrchest:$HF_TOKEN@huggingface.co/spaces/The-Ultimate-RAG-HF/The-Ultimate-RAG
+          git push --force hf2 hf2-single-commit:main
+
+      - name: Stop and remove Docker container
+        if: always()
+        run: |
+          docker stop my-app-container
+          docker rm my-app-container

.github/workflows/unit-tests.yml

@@ -0,0 +1,51 @@
+name: Unit Tests
+on:
+  pull_request:
+    branches:
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install coverage
+          pip install -r app/requirements.txt
+          pip install flake8 pytest bandit
+      - name: Run linter
+        run: |
+          flake8 app/ --max-line-length=160 --extend-ignore=E203
+      - name: Run bandit
+        run: |
+          bandit -r app -x tests
+      - name: Run unit tests with coverage
+        env:
+          HF1_URL: ${{ secrets.HF1_URL }}
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          SECRET_PEPPER: ${{ secrets.SECRET_PEPPER }}
+          JWT_ALGORITHM: ${{ secrets.JWT_ALGORITHM }}
+          PYTHONPATH: ${{ github.workspace }}
+        working-directory: ./
+        run: |
+          coverage run --source=app -m pytest app/tests/unit/test.py
+          coverage xml
+          coverage html
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-coverage-report
+          path: |
+            coverage.xml
+            htmlcov/
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml

.gitignore

@@ -0,0 +1,9 @@
+__pycache__
+/app/temp_storage
+/database
+/new_env
+/prompt.txt
+/app/key.py
+/app/env_vars.py
+/chats_storage
+/.env

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "cSpell.words": [
+        "nessacary"
+    ]
+}

CHANGELOG.md

@@ -0,0 +1,39 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [MVPv2.5] - current version
+
+- Updated README with deployment view
+- Added ID for artificial user in tests
+- Added more integration tests and proper pytest configuration
+- Accelerated file processing
+
+
+## [MVPv2] - up to 2025-07-07
+
+- Significantly reduced response time
+- Enhanced response quality and accuracy
+- Enhanced UI/UX
+- Enhanced security: confidential information now stored in .env, single .py file
+- Implemented unit and integration tests
+- Introduced CI/CD pipeline
+- Implemented response streaming (The response provided to the user when generating chunks)
+- The information now accompanies a link where it was found
+- Added JSON, CSV, and MD file types support
+- Added a draft version of user and chat separation
+- Addressed message saving bug
+- Corrected prompt to align with task requirements
+- Updated README and documentation
+
+
+## [MVPv1] - up to 2025-06-23
+
+- Established a ready-to-use RAG skeleton\
+  *(The core function allows you to attach files and ask questions about their contents. You will receive a response that includes the information, the file name, the page number, and the exact location where the information is found.)*
+- Multilingual support was added
+- Added TXT, DOC, DOCX, and PDF file types support
+- Implemented API with a simple frontend
+- Added user registration and draft authentication backend logic
+- Improved README and added License
+

Dockerfile

@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1
+FROM python:3.12.10
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+# copy and install Python reqs
+COPY app/requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+# download Qdrant binary
+RUN wget https://github.com/qdrant/qdrant/releases/download/v1.11.5/qdrant-x86_64-unknown-linux-gnu.tar.gz \
+    && tar -xzf qdrant-x86_64-unknown-linux-gnu.tar.gz \
+    && mv qdrant /home/user/.local/bin/qdrant \
+    && rm qdrant-x86_64-unknown-linux-gnu.tar.gz
+
+COPY --chown=user . /app
+
+RUN chmod +x start.sh
+
+CMD ["./start.sh"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Danil Popov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,337 @@
+---
+title: The Ultimate RAG
+emoji: 🌍
+colorFrom: pink
+colorTo: indigo
+sdk: docker
+pinned: false
+short_description: the ultimate rag
+---
+
 # The-Ultimate-RAG
-[S25] Software project for Innopolis University
+
+
+
+## Overview
+
+![logo](logo.svg)
+
+[S25] The Ultimate RAG is an Innopolis University software project that generates cited responses from a local database.
+
+## Prerequisites
+
+Before you begin, ensure the following is installed on your machine:
+
+- [Python](https://www.python.org/)
+- [Docker](https://www.docker.com/get-started/)
+
+## Installation
+
+1. **Clone the repository**
+    ```bash
+   git clone https://github.com/PopovDanil/The-Ultimate-RAG
+   cd The-Ultimate-RAG
+   ```
+2. **Set up a virtual environment (recommended)**
+
+   To isolate project dependencies and avoid conflicts, create a virtual environment:
+    - **On Unix/Linux/macOS:**
+   ```bash
+   python3 -m venv env
+   source env/bin/activate
+   ```
+    - **On Windows:**
+    ```bash
+    python -m venv env
+    env\Scripts\activate
+    ```
+3. **Install required libraries**
+
+   Within the activated virtual environment, install the dependencies:
+   ```bash
+   pip install -r ./app/requirements.txt
+   ```
+   *Note:* ensure you are in the virtual environment before running the command
+
+4. **Set up Docker**
+    - Ensure Docker is running on your machine
+    - Open a terminal, navigate to project directory, and run:
+    ```bash
+    docker-compose up --build
+    ```
+   *Note:* The initial build may take 10–20 minutes, as it needs to download large language models and other
+   dependencies.
+   Later launches will be much faster.
+
+5. **Server access**
+
+   Once the containers are running, visit `http://localhost:5050`. You should see the application’s welcome page
+
+To stop the application and shut down all containers, press `Ctrl+C` in the terminal where `docker-compose` is running,
+and then run:
+
+```bash
+   docker-compose down
+```
+
+## Usage
+
+You can try currently deployed version of the system [here](https://huggingface.co/spaces/The-Ultimate-RAG-HF/The-Ultimate-RAG). **Note**: you should use the following instructions:
+- Access the cite, you should see the *main* page with the name of the system
+- Press the button "+ Add new chat", wait until the *login* page is loaded
+- Find button "Register" (for now it is highly recommended to follow the instructions *strictly*) and press it
+- You should be redirected to *sing up* page, here you should enter your credentials (you can use Test1@test1.com in all field for testing)
+- Click **ONLY ONCE** on the button "Sign Up", and wait (for now it takes around 10 seconds to load *chat* page)
+- Now you will be able to communicate with the system
+- You can try to ask any thing and attach files. Enter a query and press the *enter* button (near the input area)
+
+## Architecture
+
+### Static view
+
+The following **diagram** depicts the current state of our codebase organization.
+![Diagram description](./docs/architecture/static-view/static-view.png)
+
+We have decided to adapt this architecture to enhance the *maintainability* of the product for the following reasons:
+- [x] A **modular** system, which is reflected by the use of subsystems in our code increases the **reusability** of the components.
+- [x] Individual subsystems can be easily **analyzed** in conjunction with monolith products.
+- [x] This approach ensures the ease and speed of **testing**.
+- [x] Additionally, each part can be **easily modified** without affecting the rest of the codebase.
+
+### Dynamic view
+
+The following **diagram** depicts the one non-trivial case of the system use: user queries the system and attach file. This diagram can halp in understating the pipeline of file processing and response generation:
+![Diagram description](./docs/architecture/dynamic-view/dynamic-view.jpeg)
+
+### Deployment view
+
+The deployment architecture of The Ultimate RAG is designed to ensure reliable, scalable, and isolated environments for testing and production.
+
+```mermaid
+graph TD
+    subgraph Test Environment
+        TEST_SPACE[Hugging Face Space <br> Test Server] -->|Connects to| TEST_DB[Test PostgreSQL Database]
+        TEST_SPACE -->|Runs| APP_TEST[Docker Container: Application]
+    end
+
+    subgraph Production Environment
+        PROD_SPACE[Hugging Face Space <br> Production Server] -->|Connects to| PROD_DB[Production PostgreSQL Database]
+        PROD_SPACE -->|Runs| APP_PROD[Docker Container: Application]
+    end
+
+    subgraph CI/CD Pipeline
+        GITHUB[GitHub Repository] -->|Push to| TEST_SPACE
+        TEST_SPACE -->|Integration Tests Pass| PROD_SPACE
+    end
+
+    subgraph External Services
+        TEST_DB -->|Hosted on| DB_SERVICE[Supabase]
+        PROD_DB -->|Hosted on| DB_SERVICE
+    end
+
+    classDef server fill:#f9f,stroke:#333,stroke-width:2px,color:#000000;
+    classDef db fill:#bbf,stroke:#333,stroke-width:2px,color:#000000;
+    classDef pipeline fill:#bfb,stroke:#333,stroke-width:2px,color:#000000;
+    class TEST_SPACE,PROD_SPACE,APP_TEST,APP_PROD server;
+    class TEST_DB,PROD_DB db;
+    class GITHUB pipeline;
+```
+
+- **Diagram Location:** The deployment diagram is stored at [`docs/architecture/deployment-view/deployment.mmd`](/docs/architecture/deployment-view/deployment.md).
+
+**Deployment Choices:**
+- **Hugging Face Spaces:** We use Hugging Face Spaces for both test and production environments due to their ease of use, free tier, and seamless integration with Git-based deployment. This allows rapid deployment and automatic scaling for our Python application.
+- **Docker:** The application is containerized using Docker (defined in `docker-compose.yml`) to ensure consistency across test and production environments, simplifying dependency management and deployment.
+- **Separate PostgreSQL Service:** The test and production PostgreSQL databases are hosted on an external service (not Hugging Face) to provide scalability, isolation, and robust database management. This ensures that test data does not interfere with production data.
+- **Isolation of Environments:** Separate Hugging Face Spaces and databases for test and production prevent test activities from affecting the live application, ensuring stability for end users.
+
+**Customer Deployment:**
+Customers can access the application directly via the production Hugging Face Space at [URL to be provided]. No local deployment is required, as the application is hosted and managed on Hugging Face. To interact with the application, customers need:
+- A web browser to access the production URL.
+- Optional: API keys or credentials (contact the [DevOps lead](https://github.com/Andrchest) for access details, if applicable).
+If customers prefer to deploy the application locally, they can follow the [Installation](#installation) instructions in this README, which include cloning the repository, setting up Docker, and configuring a `.env` file with a PostgreSQL connection string (contact the [DevOps lead](https://github.com/Andrchest) for details).
+
+## Development
+
+### Kanban board
+**Link to the board**: [Kanban board](https://github.com/orgs/The-Ultimate-RAG/projects/3/views/2)
+
+#### Column Entry Criteria
+
+##### 1. **To Do**
+- [x] Issue is created using the project’s issue templates.
+- [x] Issue is **estimated** (story points) by the Team.
+- [x] Issue is **prioritized** by the Team.
+- [x] Issue is **assigned**.
+
+##### 2. **In Progress**
+- [x] **Merge Request (MR)** is created and linked to the issue.
+- [x] **Reviewer(s)** are assigned.
+- [x] Code passes **automated checks** (unit&integration testing, linting).
+
+##### 4. **Ready to Deploy**
+- [x] MR is **approved** by at least one reviewer.
+- [x] All **review comments** are resolved.
+- [x] Code is **merged** into target branch (`main`).
+
+##### 5. **User Testing** *(Optional)*
+- [x] Feature is deployed to **testing** server.
+- [x] Testers/stakeholders are **notified**
+
+#### 6. **Done**
+- [x] Feature is deployed to **production** server.
+- [x] User testing (if needed) is **approved**.
+- [x] Issue is **closed**.
+
+### Git Workflow
+
+#### Base Workflow
+We have developed our **custom** workflow due to CI/CD integration issues and features of the development process. Key principles:
+- `main` is always deployable.
+- Feature branches are created from `main` and merged back via Pull Requests (PRs).
+- No long-lived branches except `for_testing`, which serves for deploy to the testing server.
+
+---
+
+#### Rules
+
+##### **1. Issues**
+- Use the one of the Issue Templates.
+- Include: **Description**, **Labels**, and **Milestone**.
+- Assign the most logically suitable *label* from the list of [labels](https://github.com/The-Ultimate-RAG/The-Ultimate-RAG/issues/labels) (read their description first).
+- Assign the issue to yourself, and contact [PM](https://github.com/PopovDanil) to re-assign if needed.
+
+##### **2. Branching**
+- For developing new feature create a new branch.
+- There are now strict rules for naming, but each name should logically depict the changes on code (e.g. add response streaming &rarr; response_stream).
+- For each merge mention the reason why branches were merged.
+
+##### **3. Commit Messages**
+- Template: `<type>(<scope>): <description>`.
+- Examples:
+```
+feat(auth): add login button
+fix(api): resolve null pointer in user endpoint
+```
+
+##### **4. Pull Requests (PRs) and Reviews**
+- Use the [PR Template](/.github/PULL_REQUEST_TEMPLATE/standart.md).
+- Target branch - `main`, but for testing `for_testing` can be used.
+- Contact [PM](https://github.com/PopovDanil) to assign Reviewers.
+- Merge pull request if the code passes **review**, **tests** and **linter** (in other case you will be unable to do it).
+- Delete branch after merge.
+
+##### **5. Resolving Issues**
+- Close manually only after:
+   - [x] PR is merged.
+   - [x] Feature is verified in production (if applicable).
+
+
+#### **Basic workflow example**
+```mermaid
+flowchart LR
+    A[Create Issue] --> B[Create Branch]
+    B --> C[Commit & Push]
+    C --> D[Open PR]
+    D --> E{Code Review}
+    E -->|Approved| F[Squash Merge]
+    E -->|Rejected| C
+    F --> G[Verify in Prod]
+    G --> H[Close Issue]
+
+    %% Detailed Annotations
+    subgraph "Issue Creation"
+    A
+    end
+
+    subgraph "Development"
+    B
+    C
+    end
+
+    subgraph "Collaboration"
+    D
+    E
+    end
+
+    subgraph "Release"
+    F
+    G
+    H
+    end
+```
+
+---
+
+### Secrets management
+Contact [DevOps lead](https://github.com/Andrchest) for more information.
+All the secrets are stored in `.env` file. Its content will be provided after request to DevOps lead.
+
+## Quality assurance
+
+### Quality attribute scenarios
+You can find scenarios in the [docs/quality-assurance/quality-attribute-scenarios.md](./docs/quality-assurance/quality-attribute-scenarios.md)
+
+### Automated tests
+We've implemented a comprehensive automated testing suite using the following tools:
+- 🐍 [pytest](https://docs.pytest.org/) - Primary test runner and framework
+- ⚡ [httpx](https://www.python-httpx.org/) - Async HTTP client for API testing
+
+| Test Type          | Location                      | Description                                                                 | Tools Used          |
+|--------------------|-------------------------------|-----------------------------------------------------------------------------|---------------------|
+| Unit Tests         | `app/tests/unit/`             | Tests for individual components and utility functions                       | pytest              |
+| Integration Tests  | `app/tests/integration/`      | Tests for component interactions and, API and RAG systems integrations      | pytest + httpx      |
+| Performance Tests  | `app/tests/performance/`      | *Will be added soon.* Will collect the statistical information of time, speed, and correctness evaluations  | pytest + httpx      |
+
+
+### User acceptance tests
+See [acceptance test](./docs/quality-assurance/user-acceptance-tests.md) for the formal definition of system readiness.
+
+## Build and deployment
+
+### Continuous Integration
+
+Our Continuous Integration (CI) pipeline ensures code quality by running automated checks on every pull request to the `main` branch. The pipeline is managed using GitHub Actions and is defined in:
+
+- [`.github/workflows/unit-tests.yml`](https://github.com/The-Ultimate-RAG/The-Ultimate-RAG/blob/main/.github/workflows/unit-tests.yml)
+
+In the CI pipeline, we use the following tools:
+
+- **Static Analysis Tools:**
+  - **flake8**: A linter for Python that enforces coding style and detects programming errors.
+  - **bandit**: A security vulnerability scanner for Python, identifying potential security issues in the codebase.
+
+- **Testing Tools:**
+  - **pytest**: A testing framework for Python, used to run unit tests located in [`app/tests/unit`](https://github.com/The-Ultimate-RAG/The-Ultimate-RAG/tree/main/app/tests/unit).
+
+If any checks fail, the pull request cannot be merged into `main`. All CI workflow runs can be viewed at:
+
+- [GitHub Actions - CI Workflow Runs](https://github.com/The-Ultimate-RAG/The-Ultimate-RAG/actions/workflows/unit-tests.yml)
+
+### Continuous Deployment
+
+Our Continuous Deployment (CD) pipeline automatically deploys the application after a successful merge into `main`. The pipeline is defined in:
+
+- [`.github/workflows/sync-to-hf.yml`](https://github.com/The-Ultimate-RAG/The-Ultimate-RAG/blob/main/.github/workflows/sync-to-hf.yml)
+
+The CD pipeline performs the following steps:
+1. Pushes the updated code to a **Hugging Face Space** (test environment) using `git`, where it is automatically deployed.
+2. Runs **integration tests** on the test server with a test PostgreSQL database (hosted on a separate service), using tests located in [`app/tests/integration`](https://github.com/The-Ultimate-RAG/The-Ultimate-RAG/tree/main/app/tests/integration).
+3. If integration tests pass, deploys to a separate **Hugging Face Space** (production environment) with a production PostgreSQL database (also hosted on a separate service). Deployment takes approximately 2–3 minutes.
+4. If any tests fail, the production server remains unaffected.
+
+In the CD pipeline, we use the following tools:
+
+- **Deployment Tools:**
+  - **Docker**: Builds and packages the application as a container.
+  - **git**: Pushes the application to Hugging Face Spaces for deployment.
+- **Testing Tools:**
+  - **pytest**: Runs integration tests on the test server.
+
+All CD workflow runs can be viewed at:
+
+- [GitHub Actions - CD Workflow Runs](https://github.com/The-Ultimate-RAG/The-Ultimate-RAG/actions/workflows/sync-to-hf.yml)
+
+## License
+
+This project is licensed under the [MIT License](LICENSE).

app/api/api.py

@@ -0,0 +1,353 @@
+from fastapi import (
+    FastAPI,
+    UploadFile,
+    Form,
+    File,
+    HTTPException,
+    Response,
+    Request,
+    Depends,
+)
+from fastapi.responses import (
+    FileResponse,
+    RedirectResponse,
+    StreamingResponse,
+    JSONResponse,
+)
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+
+from app.backend.controllers.users import (
+    create_user,
+    authenticate_user,
+    check_cookie,
+    clear_cookie,
+    get_current_user,
+    get_latest_chat,
+)
+from app.backend.controllers.chats import (
+    create_new_chat,
+    get_chat_with_messages,
+    update_title,
+)
+from app.backend.controllers.messages import register_message
+from app.backend.schemas import SUser
+from app.backend.models.users import User
+
+from app.core.utils import (
+    TextHandler,
+    PDFHandler,
+    protect_chat,
+    extend_context,
+    initialize_rag,
+    save_documents,
+    construct_collection_name,
+    create_collection,
+)
+from app.settings import BASE_DIR, url_user_not_required
+from app.core.document_validator import path_is_valid
+from app.core.response_parser import add_links
+from typing import Optional
+import os
+
+# TODO: implement a better TextHandler
+# TODO: optionally implement DocHandler
+
+api = FastAPI()
+
+api.mount(
+    "/chats_storage",
+    StaticFiles(directory=os.path.join(BASE_DIR, "chats_storage")),
+    name="chats_storage",
+)
+api.mount(
+    "/static",
+    StaticFiles(directory=os.path.join(BASE_DIR, "app", "frontend", "static")),
+    name="static",
+)
+templates = Jinja2Templates(
+    directory=os.path.join(BASE_DIR, "app", "frontend", "templates")
+)
+rag = initialize_rag()
+
+# NOTE: carefully read documentation to require_user
+# <--------------------------------- Middleware --------------------------------->
+"""
+Special class to have an opportunity to redirect user to login page in middleware
+"""
+
+
+class AwaitableResponse:
+    def __init__(self, response: Response):
+        self.response = response
+
+    def __await__(self):
+        yield
+        return self.response
+
+
+"""
+TODO: remove KOSTYLY -> find better way to skip requesting to login while showing pdf
+
+Middleware that requires user to log in into the system before accessing any utl
+
+NOTE: For now it is applied to all routes, but if you want to skip any, add it to the
+url_user_not_required list in settings.py (/ should be removed)
+"""
+
+
+@api.middleware("http")
+async def require_user(request: Request, call_next):
+    print(request.url.path, request.method, request.url.port)
+
+    awaitable_response = AwaitableResponse(RedirectResponse("/login", status_code=303))
+    stripped_path = request.url.path.strip("/")
+
+    if (
+            stripped_path in url_user_not_required
+            or stripped_path.startswith("pdfs")
+            or "static/styles.css" in stripped_path
+            or "favicon.ico" in stripped_path
+    ):
+        return await call_next(request)
+
+    user = get_current_user(request)
+    if user is None:
+        return await awaitable_response
+
+    response = await call_next(request)
+    return response
+
+
+# <--------------------------------- Common routes --------------------------------->
+@api.get("/health")
+async def health_check():
+    return {"status": "ok"}
+
+
+@api.get("/")
+def root(request: Request):
+    current_template = "pages/main.html"
+    return templates.TemplateResponse(
+        current_template, extend_context({"request": request})
+    )
+
+
+@api.post("/message_with_docs")
+async def send_message(
+        request: Request,
+        files: list[UploadFile] = File(None),
+        prompt: str = Form(...),
+        chat_id=Form(None),
+        user: User = Depends(get_current_user),
+) -> StreamingResponse:
+    # response = ""
+    status = 200
+    try:
+        collection_name = construct_collection_name(user, chat_id)
+
+        register_message(content=prompt, sender="user", chat_id=int(chat_id))
+
+        await save_documents(
+            collection_name, files=files, RAG=rag, user=user, chat_id=chat_id
+        )
+
+        # response = rag.generate_response_stream(collection_name=collection_name, user_prompt=prompt, stream=True)
+        # async def stream_response():
+        #     async for chunk in response:
+        #         yield chunk.json()
+
+        return StreamingResponse(
+            rag.generate_response_stream(
+                collection_name=collection_name, user_prompt=prompt, stream=True
+            ),
+            status,
+            media_type="text/event-stream",
+        )
+    except Exception as e:
+        status = 500
+        print(e)
+
+
+@api.post("/replace_message")
+async def replace_message(request: Request):
+    data = await request.json()
+    updated_message = add_links(data.get("message", ""))
+    register_message(
+        content=updated_message, sender="assistant", chat_id=int(data.get("chat_id", 0))
+    )
+    return JSONResponse({"updated_message": updated_message})
+
+
+@api.get("/viewer")
+def show_document(
+        request: Request,
+        path: str,
+        page: Optional[int] = 1,
+        lines: Optional[str] = "1-1",
+        start: Optional[int] = 0,
+):
+    if not path_is_valid(path):
+        return HTTPException(status_code=404, detail="Document not found")
+
+    ext = path.split(".")[-1]
+    if ext == "pdf":
+        return PDFHandler(request, path=path, page=page, templates=templates)
+    elif ext in ("txt", "csv", "md", "json"):
+        return TextHandler(request, path=path, lines=lines, templates=templates)
+    elif ext in ("docx", "doc"):
+        return TextHandler(
+            request, path=path, lines=lines, templates=templates
+        )  # should be a bit different handler
+    else:
+        return FileResponse(path=path)
+
+
+# <--------------------------------- Get --------------------------------->
+@api.get("/new_user")
+def new_user_post(request: Request):
+    current_template = "pages/registration.html"
+    return templates.TemplateResponse(
+        current_template, extend_context({"request": request})
+    )
+
+
+@api.get("/login")
+def login_get(request: Request):
+    current_template = "pages/login.html"
+    return templates.TemplateResponse(
+        current_template, extend_context({"request": request})
+    )
+
+
+@api.get("/cookie_test")
+def test_cookie(request: Request):
+    return check_cookie(request)
+
+
+"""
+Use only for testing. For now, provides user info for logged ones, and redirects to
+login in other case
+"""
+
+
+@api.get("/test")
+def test(request: Request, user: User = Depends(get_current_user)):
+    return {
+        "user": {
+            "email": user.email,
+            "password_hash": user.password_hash,
+            # "chats": user.chats, # Note: it will rise error since due to the optimization associated fields are not loaded
+            # it is just a reference, but the session is closed, however you are trying to get access to the data through this session
+        }
+    }
+
+
+@api.post("/chats/id={chat_id}/history")
+def show_chat_history(request: Request, chat_id: int):
+    chat = get_chat_with_messages(chat_id)
+    user = get_current_user(request)
+
+    update_title(chat["chat_id"])
+
+    if not protect_chat(user, chat_id):
+        raise HTTPException(401, "Yod do not have rights to use this chat!")
+
+    context = chat
+
+    return context
+
+
+@api.get("/chats/id={chat_id}")
+def show_chat(request: Request, chat_id: int):
+    current_template = "pages/chat.html"
+
+    chat = get_chat_with_messages(chat_id)
+    user = get_current_user(request)
+
+    update_title(chat["chat_id"])
+
+    if not protect_chat(user, chat_id):
+        raise HTTPException(401, "Yod do not have rights to use this chat!")
+
+    context = extend_context({"request": request, "user": user}, selected=chat_id)
+    context.update(chat)
+
+    return templates.TemplateResponse(current_template, context)
+
+
+@api.get("/logout")
+def logout(response: Response):
+    return clear_cookie(response)
+
+
+@api.get("/last_user_chat")
+def last_user_chat(request: Request, user: User = Depends(get_current_user)):
+    chat = get_latest_chat(user)
+    url = None
+
+    if chat is None:
+        print("new_chat")
+        new_chat = create_new_chat("new chat", user)
+        url = new_chat.get("url")
+
+        try:
+            create_collection(user, new_chat.get("chat_id"), rag)
+        except Exception as e:
+            raise HTTPException(500, e)
+
+    else:
+        url = f"/chats/id={chat.id}"
+
+    return RedirectResponse(url, status_code=303)
+
+
+# <--------------------------------- Post --------------------------------->
+@api.post("/new_user")
+def new_user(response: Response, user: SUser):
+    return create_user(response, user.email, user.password)
+
+
+class LoginData(BaseModel):
+    email: str
+    password: str
+
+
+@api.post("/login")
+def login_post(response: Response, user_data: LoginData):
+    try:
+        # Validate the user data against the SUser schema for regular users
+        # This enforces email format and password complexity for non-admins
+        user_schema = SUser(email=user_data.email, password=user_data.password)
+    except ValueError as e:
+        # If validation fails, return a detailed error
+        raise HTTPException(status_code=422, detail=f"Validation error: {e}")
+
+    # If validation passes, proceed with the standard authentication process
+    return authenticate_user(response, user_schema.email, user_schema.password)
+
+
+@api.post("/new_chat")
+def create_chat(
+        request: Request,
+        title: Optional[str] = "new chat",
+        user: User = Depends(get_current_user),
+):
+    new_chat = create_new_chat(title, user)
+    url = new_chat.get("url")
+    chat_id = new_chat.get("chat_id")
+
+    if url is None or chat_id is None:
+        raise HTTPException(500, "New chat was not created")
+
+    try:
+        create_collection(user, chat_id, rag)
+    except Exception as e:
+        raise HTTPException(500, e)
+
+    return RedirectResponse(url, status_code=303)
+
+
+if __name__ == "__main__":
+    pass

app/automigration.py

@@ -0,0 +1,4 @@
+from app.backend.models.db_service import automigrate
+
+if __name__ == "__main__":
+    automigrate()

app/backend/controllers/base_controller.py

@@ -0,0 +1,5 @@
+from sqlalchemy import create_engine
+from app.settings import settings
+
+postgres_config = settings.postgres.model_dump()
+engine = create_engine(**postgres_config)

app/backend/controllers/chats.py

@@ -0,0 +1,113 @@
+from app.backend.models.users import User, get_user_chats
+from app.backend.models.chats import (
+    new_chat,
+    get_chat_by_id,
+    get_chats_by_user_id,
+    refresh_title,
+)
+from app.backend.models.messages import get_messages_by_chat_id, Message
+from app.settings import BASE_DIR
+from fastapi import HTTPException
+from datetime import datetime, timedelta
+import os
+
+
+def create_new_chat(title: str | None, user: User) -> dict:
+    chat_id = new_chat(title, user)
+    try:
+        path_to_chat = os.path.join(
+            BASE_DIR,
+            "chats_storage",
+            f"user_id={user.id}",
+            f"chat_id={chat_id}",
+            "documents",
+        )
+        os.makedirs(path_to_chat, exist_ok=True)
+    except Exception as e:
+        print(e)
+        raise HTTPException(500, "Unable to create a new chat")
+
+    return {"url": f"/chats/id={chat_id}", "chat_id": chat_id}
+
+
+def dump_messages_dict(messages: list[Message], dst: dict) -> None:
+    history = []
+
+    for message in messages:
+        history.append({"role": message.sender, "content": message.content})
+        print(message.sender, message.content[:100])
+
+    dst.update({"history": history})
+
+
+def get_chat_with_messages(id: int) -> dict:
+    response = {"chat_id": id}
+
+    chat = get_chat_by_id(id=id)
+    if chat is None:
+        raise HTTPException(418, f"Invalid chat id. Chat with id={id} does not exists!")
+
+    messages = get_messages_by_chat_id(id=id)
+    dump_messages_dict(messages, response)
+
+    return response
+
+
+def create_dict_from_chat(chat) -> dict:
+    return {"id": chat.id, "title": chat.title}
+
+
+def list_user_chats(user_id: int) -> list[dict]:
+    current_date = datetime.now()
+
+    today = []
+    last_week = []
+    last_month = []
+    other = []
+
+    chats = get_chats_by_user_id(user_id)
+    for chat in chats:
+        if current_date - timedelta(days=1) <= chat.created_at:
+            today.append(chat)
+        elif current_date - timedelta(weeks=1) <= chat.created_at:
+            last_week.append(chat)
+        elif current_date - timedelta(weeks=4) <= chat.created_at:
+            last_month.append(chat)
+        else:
+            other.append(chat)
+
+    result = []
+
+    # da da eto ochen ploho ...
+    if len(today):
+        result.append(
+            {"title": "TODAY", "chats": [create_dict_from_chat(chat) for chat in today]}
+        )
+    if len(last_week):
+        result.append(
+            {
+                "title": "LAST WEEK",
+                "chats": [create_dict_from_chat(chat) for chat in last_week],
+            }
+        )
+    if len(last_month):
+        result.append(
+            {
+                "title": "LAST MONTH",
+                "chats": [create_dict_from_chat(chat) for chat in last_month],
+            }
+        )
+    if len(other):
+        result.append(
+            {"title": "LATER", "chats": [create_dict_from_chat(chat) for chat in other]}
+        )
+
+    return result
+
+
+def verify_ownership_rights(user: User, chat_id: int) -> bool:
+    return chat_id in [chat.id for chat in get_user_chats(user)]
+
+
+def update_title(chat_id: int) -> bool:
+    return refresh_title(chat_id)

app/backend/controllers/messages.py

@@ -0,0 +1,18 @@
+import re
+
+from app.backend.models.messages import new_message
+
+
+def remove_html_tags(content: str) -> str:
+    pattern = "<(.*?)>"
+    replace_with = (
+        "<a href=https://www.youtube.com/results?search_query=rickroll>click me</a>"
+    )
+    de_taggeed = re.sub(pattern, "REPLACE_WITH_RICKROLL", content)
+
+    return de_taggeed.replace("REPLACE_WITH_RICKROLL", replace_with)
+
+
+def register_message(content: str, sender: str, chat_id: int) -> None:
+    message = content if sender == "assistant" else remove_html_tags(content)
+    return new_message(chat_id=chat_id, sender=sender, content=message)

app/backend/controllers/user_controller.py

@@ -0,0 +1,113 @@
+import logging
+from datetime import datetime
+from typing import Optional
+from sqlalchemy.exc import SQLAlchemyError, IntegrityError
+from sqlalchemy.orm import Session
+
+from app.backend.schemas import LanguageOptions, ThemeOptions
+from app.backend.exceptions import (
+    DatabaseError,
+    UserNotFoundError,
+    UserAlreadyExistsError,
+)
+from app.backend.models.users import User
+
+
+class UserController:
+    def __init__(self, database_session: Session):
+        self.database = database_session
+
+    @staticmethod
+    def _execute_query(query) -> Optional[User]:
+        """
+        Helper method to execute a query and handle common database errors.
+        """
+        try:
+            return query.first()
+        except SQLAlchemyError as e:
+            logging.error(f"Database error during user query: {e}", exc_info=True)
+            raise DatabaseError(f"Failed to query user due to a database error: {e}")
+
+    def add_new_user(
+        self, email: str, password_hash: str, access_string_hash: str
+    ) -> User:
+        if self.find_user_by_email(email):
+            logging.warning(f"Attempted to register existing email: {email}")
+            raise UserAlreadyExistsError(f"User with email {email} already registered")
+
+        new_user = User(
+            email=email,
+            password_hash=password_hash,
+            access_string_hash=access_string_hash,
+        )
+
+        self.database.add(new_user)
+
+        try:
+            self.database.commit()
+            self.database.refresh(new_user)
+            logging.info(f"Successfully registered new user: {new_user}")
+            return new_user
+        except IntegrityError as e:
+            self.database.rollback()
+            logging.error(
+                f"Integrity error when adding user '{email}': {e}", exc_info=True
+            )
+            raise UserAlreadyExistsError(f"User with email {email} already exists")
+        except SQLAlchemyError as e:
+            self.database.rollback()
+            raise DatabaseError(f"Failed to add new user due to a database error: {e}")
+
+    def find_user_by_id(self, user_id: int) -> User | None:
+        query = self.database.query(User).filter(User.id == user_id)
+        return self._execute_query(query)
+
+    def find_user_by_email(self, email: str) -> User | None:
+        query = self.database.query(User).filter(User.email == email)
+        return self._execute_query(query)
+
+    def find_user_by_access_string(self, access_string_hash: str) -> User | None:
+        query = self.database.query(User).filter(
+            User.access_string_hash == access_string_hash
+        )
+        return self._execute_query(query)
+
+    def update_user(self, user_id: int, **kwargs) -> User:
+        user_to_update = self.find_user_by_id(user_id)
+
+        if not user_to_update:
+            raise UserNotFoundError("User not found")
+
+        allowed_updates = {
+            "language": LanguageOptions,
+            "theme": ThemeOptions,
+            "access_string_hash": str,
+            "password_hash": str,
+            "reset_token_expires_at": datetime,
+        }
+
+        for key, value in kwargs.items():
+            if key in allowed_updates:
+                expected_type = allowed_updates[key]
+                if not isinstance(value, expected_type) or value is None:
+                    raise ValueError(
+                        f"Invalid type for {key}. Expected {expected_type}, got {type(value).__name__}"
+                    )
+
+                setattr(user_to_update, key, value)
+            else:
+                logging.warning(
+                    f"Attempted to updated disallowed key: {key} for user {user_id}. Ignoring"
+                )
+
+        try:
+            self.database.commit()
+            self.database.refresh(user_to_update)
+            logging.info(f"Successfully updated user: {user_to_update}")
+            return user_to_update
+        except SQLAlchemyError as e:
+            logging.error(
+                f"Failed to update user ID {user_id} due to a database error: {e}",
+                exc_info=True,
+            )
+            raise DatabaseError(f"Failed to update user due to a database error: {e}")

app/backend/controllers/users.py

@@ -0,0 +1,197 @@
+from app.backend.models.users import (
+    User,
+    add_new_user,
+    find_user_by_email,
+    find_user_by_access_string,
+    update_user,
+    get_user_last_chat,
+)
+from app.backend.models.chats import Chat
+from bcrypt import gensalt, hashpw, checkpw
+from app.settings import settings
+from fastapi import HTTPException
+import jwt
+from datetime import datetime, timedelta
+from fastapi import Response, Request
+from secrets import token_urlsafe
+import hmac
+import hashlib
+
+# A vot nado bilo izuchat kak web dev rabotaet
+
+"""
+Creates a jwt token by access string
+
+Param:
+access_string - randomly (safe methods) generated string (by default - 16 len)
+expires_delta - time in seconds, defines a token lifetime
+
+Returns:
+string with 4 sections (valid jwt token)
+"""
+
+
+def create_access_token(
+    access_string: str, expires_delta: timedelta = settings.max_cookie_lifetime
+) -> str:
+    token_payload = {
+        "access_string": access_string,
+    }
+
+    token_payload.update({"exp": datetime.now() + expires_delta})
+    encoded_jwt: str = jwt.encode(
+        token_payload, settings.secret_pepper, algorithm=settings.jwt_algorithm.replace("\r", "")
+    )
+
+    return encoded_jwt
+
+
+"""
+Safely creates random string of 16 chars
+"""
+
+
+def create_access_string() -> str:
+    return token_urlsafe(16)
+
+
+"""
+Hashes access string using hmac and sha256
+
+We can not use the same methods as we do to save password
+since we need to know a salt to get similar hash, but since
+we put a raw string (non-hashed) we won't be able to guess
+salt
+"""
+
+
+def hash_access_string(string: str) -> str:
+    return hmac.new(
+        key=str(settings.secret_pepper).encode("utf-8"),
+        msg=string.encode("utf-8"),
+        digestmod=hashlib.sha256,
+    ).hexdigest()
+
+
+"""
+Creates a new user and sets a cookie with jwt token
+
+Params:
+response - needed to set a cookie
+...
+
+Returns:
+Dict to send a response in JSON
+"""
+
+
+def create_user(response: Response, email: str, password: str) -> dict:
+    user: User = find_user_by_email(email=email)
+    if user is not None:
+        return HTTPException(418, "The user with similar email already exists")
+
+    salt: bytes = gensalt(rounds=16)
+    password_hashed: str = hashpw(password.encode("utf-8"), salt).decode("utf-8")
+
+    access_string: str = create_access_string()
+    access_string_hashed: str = hash_access_string(string=access_string)
+
+    id = add_new_user(
+        email=email,
+        password_hash=password_hashed,
+        access_string_hash=access_string_hashed,
+    )
+
+    print(id)
+
+    access_token: str = create_access_token(access_string=access_string)
+    response.set_cookie(
+        key="access_token",
+        value=access_token,
+        path="/",
+        max_age=settings.max_cookie_lifetime,
+        httponly=True,
+    )
+
+    return {"status": "ok", "id": id if id is not None else 0}
+
+
+"""
+Finds user by email. If user is found, sets a cookie with token
+"""
+
+
+def authenticate_user(response: Response, email: str, password: str) -> dict:
+    user: User = find_user_by_email(email=email)
+
+    if not user:
+        raise HTTPException(418, "User does not exists")
+
+    if not checkpw(password.encode("utf-8"), user.password_hash.encode("utf-8")):
+        raise HTTPException(418, "Wrong credentials")
+
+    access_string: str = create_access_string()
+    access_string_hashed: str = hash_access_string(string=access_string)
+
+    update_user(user, access_string_hash=access_string_hashed)
+
+    access_token = create_access_token(access_string)
+    response.set_cookie(
+        key="access_token",
+        value=access_token,
+        path="/",
+        max_age=settings.max_cookie_lifetime,
+        httponly=True,
+    )
+
+    return {"status": "ok"}
+
+
+"""
+Get user from token stored in cookies
+"""
+
+
+def get_current_user(request: Request) -> User | None:
+    user = None
+    token: str | None = request.cookies.get("access_token")
+    if not token:
+        return None
+
+    try:
+        access_string = jwt.decode(
+            jwt=bytes(token, encoding="utf-8"),
+            key=settings.secret_pepper,
+            algorithms=[settings.jwt_algorithm.replace("\r", "")],
+        ).get("access_string")
+
+        user = find_user_by_access_string(hash_access_string(access_string))
+    except Exception as e:
+        print(e)
+
+    if not user:
+        return None
+
+    return user
+
+
+"""
+Checks if cookie with access token is present
+"""
+
+
+def check_cookie(request: Request) -> dict:
+    result = {"token": "No token is present"}
+    token = request.cookies.get("access_token")
+    if token:
+        result["token"] = token
+    return result
+
+
+def clear_cookie(response: Response) -> dict:
+    response.set_cookie(key="access_token", value="", httponly=True)
+    return {"status": "ok"}
+
+
+def get_latest_chat(user: User) -> Chat | None:
+    return get_user_last_chat(user)

app/backend/models/base_model.py

@@ -0,0 +1,10 @@
+from sqlalchemy import Column, DateTime
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.sql import func
+
+
+class Base(DeclarativeBase):
+    __abstract__ = True
+    created_at = Column("created_at", DateTime, default=func.now())
+    deleted_at = Column("deleted_at", DateTime, nullable=True)
+    updated_at = Column("updated_at", DateTime, nullable=True)

app/backend/models/chats.py

@@ -0,0 +1,54 @@
+from app.backend.models.base_model import Base
+from sqlalchemy import Integer, String, Column, ForeignKey
+from sqlalchemy.orm import relationship, Session
+from app.backend.controllers.base_controller import engine
+
+
+class Chat(Base):
+    __tablename__ = "chats"
+    id = Column("id", Integer, autoincrement=True, primary_key=True, unique=True)
+    title = Column("title", String, nullable=True)
+    user_id = Column(Integer, ForeignKey("users.id"))
+    user = relationship("User", back_populates="chats")
+    messages = relationship("Message", back_populates="chat")
+
+
+def new_chat(title: str | None, user) -> int:
+    id = None
+    with Session(autoflush=False, bind=engine) as db:
+        user = db.merge(user)
+        new_chat = Chat(user_id=user.id, user=user)
+        if title:
+            new_chat.title = title
+        db.add(new_chat)
+        db.commit()
+        id = new_chat.id
+    return id
+
+
+def get_chat_by_id(id: int) -> Chat | None:
+    with Session(autoflush=False, bind=engine) as db:
+        return db.query(Chat).where(Chat.id == id).first()
+
+
+def get_chats_by_user_id(id: int) -> list[Chat]:
+    with Session(autoflush=False, bind=engine) as db:
+        return (
+            db.query(Chat).filter(Chat.user_id == id).order_by(Chat.created_at.desc())
+        )
+
+
+def refresh_title(chat_id: int) -> bool:
+    with Session(autoflush=False, bind=engine) as db:
+        chat = db.get(Chat, chat_id)
+        messages = chat.messages
+
+        if messages is None or len(messages) == 0:
+            return False
+
+        chat.title = messages[0].content[:47]
+        if len(messages[0].content) > 46:
+            chat.title += "..."
+
+        db.commit()
+        return True

app/backend/models/db_service.py

@@ -0,0 +1,30 @@
+from app.backend.controllers.base_controller import engine
+from app.backend.models.base_model import Base
+from app.backend.models.chats import Chat
+from app.backend.models.messages import Message
+from app.backend.models.users import User
+
+
+def table_exists(name: str) -> bool:
+    return engine.dialect.has_table(engine, name)
+
+
+def create_tables() -> None:
+    Base.metadata.create_all(engine)
+
+
+def drop_tables() -> None:
+    # for now the order matters, so
+    # TODO: add cascade deletion for models
+    Message.__table__.drop(engine)
+    Chat.__table__.drop(engine)
+    User.__table__.drop(engine)
+
+
+def automigrate() -> None:
+    try:
+        drop_tables()
+    except Exception as e:
+        print(e)
+
+    create_tables()

app/backend/models/messages.py

@@ -0,0 +1,25 @@
+from sqlalchemy import Column, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import Session, relationship
+
+from app.backend.controllers.base_controller import engine
+from app.backend.models.base_model import Base
+
+
+class Message(Base):
+    __tablename__ = "messages"
+    id = Column("id", Integer, autoincrement=True, primary_key=True, unique=True)
+    content = Column("text", Text)
+    sender = Column("role", String)
+    chat_id = Column(Integer, ForeignKey("chats.id"))
+    chat = relationship("Chat", back_populates="messages")
+
+
+def new_message(chat_id: int, sender: str, content: str):
+    with Session(autoflush=False, bind=engine) as db:
+        db.add(Message(content=content, sender=sender, chat_id=chat_id))
+        db.commit()
+
+
+def get_messages_by_chat_id(id: int) -> list[Message]:
+    with Session(autoflush=False, bind=engine) as db:
+        return db.query(Message).filter(Message.chat_id == id)

app/backend/models/users.py

@@ -0,0 +1,80 @@
+from sqlalchemy import Column, String, Integer
+from sqlalchemy.orm import relationship, Session
+
+from app.backend.models.base_model import Base
+from app.backend.controllers.base_controller import engine
+from app.backend.models.chats import Chat
+
+
+class User(Base):
+    __tablename__ = "users"
+    id = Column("id", Integer, autoincrement=True, primary_key=True, unique=True)
+    email = Column("email", String, unique=True, nullable=False)
+    password_hash = Column("password_hash", String, nullable=False)
+    language = Column("language", String, default="English", nullable=False)
+    theme = Column("theme", String, default="light", nullable=False)
+    access_string_hash = Column("access_string_hash", String, nullable=True)
+    chats = relationship("Chat", back_populates="user")
+
+
+def add_new_user(email: str, password_hash: str, access_string_hash: str) -> int | None:
+    with Session(autoflush=False, bind=engine, expire_on_commit=False) as db:
+        user = User(
+            email=email,
+            password_hash=password_hash,
+            access_string_hash=access_string_hash,
+        )
+        db.add(user)
+        db.commit()
+        db.refresh(user)
+        return user.id
+
+
+def find_user_by_id(id: int) -> User | None:
+    with Session(autoflush=False, bind=engine) as db:
+        return db.query(User).where(User.id == id).first()
+
+
+def find_user_by_email(email: str) -> User | None:
+    with Session(autoflush=False, bind=engine) as db:
+        return db.query(User).where(User.email == email).first()
+
+
+def find_user_by_access_string(access_string_hash: str) -> User | None:
+    with Session(autoflush=False, bind=engine, expire_on_commit=False) as db:
+        user = (
+            db.query(User).where(User.access_string_hash == access_string_hash).first()
+        )
+        return user
+
+
+def update_user(
+    user: User, language: str = None, theme: str = None, access_string_hash: str = None
+) -> None:
+    with Session(autoflush=False, bind=engine) as db:
+        user = db.merge(user)
+        if language:
+            user.language = language
+        if theme:
+            user.theme = theme
+        if access_string_hash:
+            user.access_string_hash = access_string_hash
+        db.commit()
+
+
+def get_user_chats(user: User) -> list[Chat]:
+    with Session(autoflush=False, bind=engine) as db:
+        user = db.get(User, user.id)
+        return user.chats
+
+
+def get_user_last_chat(user: User) -> Chat | None:
+    with Session(autoflush=False, bind=engine) as db:
+        user = db.get(User, user.id)
+
+        chats = user.chats
+
+        if chats is not None and len(chats):
+            return chats[-1]
+
+        return None

app/backend/schemas.py

@@ -0,0 +1,45 @@
+from enum import Enum
+from pydantic import BaseModel, Field, EmailStr, field_validator
+import re
+
+
+class ThemeOptions(str, Enum):
+    LIGHT = "light"
+    DARK = "dark"
+
+
+class LanguageOptions(str, Enum):
+    AR = "ar"
+    EN = "en"
+    RU = "ru"
+
+
+class SUser(BaseModel):
+    email: EmailStr
+    password: str = Field(default=..., min_length=8, max_length=32)
+
+    @field_validator("password", mode="before")
+    def validate_password(cls, password_to_validate):
+        """
+        Validates the strength of the password.
+
+        The password **must** contain:
+            - At least one digit
+            - At least one special character
+            - At least one uppercase character
+            - At least one lowercase character
+        """
+
+        if not re.search(r"\d", password_to_validate):
+            raise ValueError("Password must contain at least one number.")
+
+        if not re.search(r"[!@#$%^&*()_+\-=\[\]{};:\'\",.<>?`~]", password_to_validate):
+            raise ValueError("Password must contain at least one special symbol.")
+
+        if not re.search(r"[A-Z]", password_to_validate):
+            raise ValueError("Password must contain at least one uppercase letter.")
+
+        if not re.search(r"[a-z]", password_to_validate):
+            raise ValueError("Password must contain at least one lowercase letter.")
+
+        return password_to_validate

app/core/chunks.py

@@ -0,0 +1,54 @@
+import uuid
+
+
+class Chunk:
+    """
+    id -> unique number in uuid format, can be tried https://www.uuidgenerator.net/
+    start_index -> the index of the first char from the beginning of the original document
+
+    TODO: implement access modifiers and set of getters and setters
+    """
+
+    def __init__(
+        self,
+        id: uuid.UUID,
+        filename: str,
+        page_number: int,
+        start_index: int,
+        start_line: int,
+        end_line: int,
+        text: str,
+    ):
+        self.id: uuid.UUID = id
+        self.filename: str = filename
+        self.page_number: int = page_number
+        self.start_index: int = start_index
+        self.start_line: int = start_line
+        self.end_line: int = end_line
+        self.text: str = text
+
+    def get_raw_text(self) -> str:
+        return self.text
+
+    def get_splitted_text(self) -> list[str]:
+        return self.text.split(" ")
+
+    def get_metadata(self) -> dict:
+        return {
+            "id": self.id,
+            "filename": self.filename,
+            "page_number": self.page_number,
+            "start_index": self.start_index,
+            "start_line": self.start_line,
+            "end_line": self.end_line,
+        }
+
+    # TODO: remove kostyly
+    def __str__(self):
+        return (
+            f"Chunk from {self.filename.split('/')[-1]}, "
+            f"page - {self.page_number}, "
+            f"start - {self.start_line}, "
+            f"end - {self.end_line}, "
+            f"and text - {self.text[:100]}... ({len(self.text)})\n"
+        )

app/core/database.py

@@ -0,0 +1,217 @@
+from qdrant_client import QdrantClient  # main component to provide the access to db
+from qdrant_client.http.models import (
+    ScoredPoint,
+    Filter,
+    FieldCondition,
+    MatchText
+)
+from qdrant_client.models import (
+    VectorParams,
+    Distance,
+    PointStruct,
+    TextIndexParams,
+    TokenizerType
+)  # VectorParams -> config of vectors that will be used as primary keys
+from app.core.models import Embedder  # Distance -> defines the metric
+from app.core.chunks import Chunk  # PointStruct -> instance that will be stored in db
+import numpy as np
+from uuid import UUID
+from app.settings import settings
+import time
+from fastapi import HTTPException
+import re
+
+
+class VectorDatabase:
+    def __init__(self, embedder: Embedder, host: str = "qdrant", port: int = 6333):
+        self.host: str = host
+        self.client: QdrantClient = self._initialize_qdrant_client()
+        self.embedder: Embedder = embedder  # embedder is used to convert a user's query
+        self.already_stored: np.array[np.array] = np.array([]).reshape(
+            0, embedder.get_vector_dimensionality()
+        )
+
+    def store(
+        self, collection_name: str, chunks: list[Chunk], batch_size: int = 1000
+    ) -> None:
+        points: list[PointStruct] = []
+
+        vectors = self.embedder.encode([chunk.get_raw_text() for chunk in chunks])
+
+        for vector, chunk in zip(vectors, chunks):
+            if self.accept_vector(collection_name, vector):
+                points.append(
+                    PointStruct(
+                        id=str(chunk.id),
+                        vector=vector,
+                        payload={
+                            "metadata": chunk.get_metadata(),
+                            "text": chunk.get_raw_text(),
+                        },
+                    )
+                )
+
+        if len(points):
+            for group in range(0, len(points), batch_size):
+                self.client.upsert(
+                    collection_name=collection_name,
+                    points=points[group : group + batch_size],
+                    wait=False,
+                )
+
+    """
+    Measures a cosine of angle between tow vectors
+    """
+
+    def cosine_similarity(self, vec1, vec2):
+        vec1_np = np.array(vec1)
+        vec2_np = np.array(vec2)
+        return vec1_np @ vec2_np / (np.linalg.norm(vec1_np) * np.linalg.norm(vec2_np))
+
+    """
+    Defines weather the vector should be stored in the db by searching for the most
+    similar one
+    """
+
+    def accept_vector(self, collection_name: str, vector: np.array) -> bool:
+        most_similar = self.client.query_points(
+            collection_name=collection_name, query=vector, limit=1, with_vectors=True
+        ).points
+
+        if not len(most_similar):
+            return True
+        else:
+            most_similar = most_similar[0]
+
+        if 1 - self.cosine_similarity(vector, most_similar.vector) < settings.max_delta:
+            return False
+        return True
+
+    def construct_keywords_list(self, query: str) -> list[FieldCondition]:
+        keywords = re.findall(r'\b[A-Z]{2,}\b', query)
+        filters = []
+
+        print(keywords)
+
+        for word in keywords:
+            if len(word) > 30 or len(word) < 2:
+                continue
+            filters.append(FieldCondition(key="text", match=MatchText(text=word)))
+
+        return filters
+
+    """
+    According to tests, re-ranker needs ~7-10 chunks to generate the most accurate hit
+
+    TODO: implement hybrid search
+    """
+
+    def search(self, collection_name: str, query: str, top_k: int = 5) -> list[Chunk]:
+        query_embedded: np.ndarray = self.embedder.encode(query)
+
+        if isinstance(query_embedded, list):
+            query_embedded = query_embedded[0]
+
+        keywords = self.construct_keywords_list(query)
+
+        dense_result: list[ScoredPoint] = self.client.query_points(
+            collection_name=collection_name, query=query_embedded, limit=int(top_k * 0.7)
+        ).points
+
+        sparse_result: list[ScoredPoint] = self.client.query_points(
+            collection_name=collection_name, query=query_embedded, limit=int(top_k * 0.3),
+            query_filter=Filter(should=keywords)
+        ).points
+
+        combined = [*dense_result, *sparse_result]
+
+        print(len(combined))
+
+        return [
+            Chunk(
+                id=UUID(point.payload.get("metadata", {}).get("id", "")),
+                filename=point.payload.get("metadata", {}).get("filename", ""),
+                page_number=point.payload.get("metadata", {}).get("page_number", 0),
+                start_index=point.payload.get("metadata", {}).get("start_index", 0),
+                start_line=point.payload.get("metadata", {}).get("start_line", 0),
+                end_line=point.payload.get("metadata", {}).get("end_line", 0),
+                text=point.payload.get("text", ""),
+            )
+            for point in combined
+        ]
+
+    def _initialize_qdrant_client(self, max_retries=5, delay=2) -> QdrantClient:
+        for attempt in range(max_retries):
+            try:
+                client = QdrantClient(**settings.qdrant.model_dump())
+                client.get_collections()
+                return client
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    raise HTTPException(
+                        500,
+                        f"Failed to connect to Qdrant server after {max_retries} attempts. "
+                        f"Last error: {str(e)}",
+                    )
+
+                print(
+                    f"Connection attempt {attempt + 1} out of {max_retries} failed. "
+                    f"Retrying in {delay} seconds..."
+                )
+
+                time.sleep(delay)
+                delay *= 2
+
+    def _check_collection_exists(self, collection_name: str) -> bool:
+        try:
+            return self.client.collection_exists(collection_name)
+        except Exception as e:
+            raise HTTPException(
+                500,
+                f"Failed to check collection {collection_name} exists. Last error: {str(e)}",
+            )
+
+    def _create_collection(self, collection_name: str) -> None:
+        try:
+            self.client.create_collection(
+                collection_name=collection_name,
+                vectors_config=VectorParams(
+                    size=self.embedder.get_vector_dimensionality(),
+                    distance=Distance.COSINE,
+                ),
+            )
+            self.client.create_payload_index(
+                collection_name=collection_name,
+                field_name="text",
+                field_schema=TextIndexParams(
+                    type="text",
+                    tokenizer=TokenizerType.WORD,
+                    min_token_len=2,
+                    max_token_len=30,
+                    lowercase=True
+                )
+            )
+        except Exception as e:
+            raise HTTPException(
+                500, f"Failed to create collection {self.collection_name}: {str(e)}"
+            )
+
+    def create_collection(self, collection_name: str) -> None:
+        try:
+            if self._check_collection_exists(collection_name):
+                return
+            self._create_collection(collection_name)
+        except Exception as e:
+            print(e)
+            raise HTTPException(500, e)
+
+    def __del__(self):
+        if hasattr(self, "client"):
+            self.client.close()
+
+    def get_collections(self) -> list[str]:
+        try:
+            return self.client.get_collections()
+        except Exception as e:
+            print(e)
+            raise HTTPException(500, "Failed to get collection names")

app/core/document_validator.py

@@ -0,0 +1,9 @@
+import os
+
+"""
+Checks if the given path is valid and file exists
+"""
+
+
+def path_is_valid(path: str) -> bool:
+    return os.path.exists(path)

app/core/main.py

@@ -0,0 +1,33 @@
+from app.settings import settings, BASE_DIR
+import uvicorn
+import os
+from app.backend.models.db_service import automigrate
+
+
+def initialize_system() -> bool:
+    path = BASE_DIR
+    chats_storage_path = os.path.join(path, "chats_storage")
+    database_path = os.path.join(path, "database")
+
+    try:
+        os.makedirs(database_path, exist_ok=True)
+        os.makedirs(chats_storage_path, exist_ok=True)
+    except Exception:
+        raise RuntimeError("Not all required directories were initialized")
+
+    try:
+        # os.system(f"pip install -r {os.path.join(base_path, 'requirements.txt')}")
+        pass
+    except Exception:
+        raise RuntimeError("Not all package were downloaded")
+
+
+def main():
+    automigrate()  # Note: it will drop all existing dbs and create a new ones
+    initialize_system()
+    uvicorn.run(**settings.api.model_dump())
+
+
+if __name__ == "__main__":
+    # ATTENTION: run from base dir ---> python -m app.main
+    main()

app/core/models.py

@@ -0,0 +1,203 @@
+import os
+
+from dotenv import load_dotenv
+from sentence_transformers import (
+    SentenceTransformer,
+    CrossEncoder,
+)  # SentenceTransformer -> model for embeddings, CrossEncoder -> re-ranker
+from ctransformers import AutoModelForCausalLM
+from torch import Tensor
+from google import genai
+from google.genai import types
+from app.core.chunks import Chunk
+from app.settings import settings, BASE_DIR, GeminiEmbeddingSettings
+
+load_dotenv()
+
+
+class Embedder:
+    def __init__(self, model: str = "BAAI/bge-m3"):
+        self.device: str = settings.device
+        self.model_name: str = model
+        self.model: SentenceTransformer = SentenceTransformer(model, device=self.device)
+
+    """
+    Encodes string to dense vector
+    """
+
+    def encode(self, text: str | list[str]) -> Tensor | list[Tensor]:
+        return self.model.encode(sentences=text, show_progress_bar=False, batch_size=32)
+
+    """
+    Returns the dimensionality of dense vector
+    """
+
+    def get_vector_dimensionality(self) -> int | None:
+        return self.model.get_sentence_embedding_dimension()
+
+
+class Reranker:
+    def __init__(self, model: str = "cross-encoder/ms-marco-MiniLM-L6-v2"):
+        self.device: str = settings.device
+        self.model_name: str = model
+        self.model: CrossEncoder = CrossEncoder(model, device=self.device)
+
+    """
+    Returns re-sorted (by relevance) vector with dicts, from which we need only the 'corpus_id'
+    since it is a position of chunk in original list
+    """
+
+    def rank(self, query: str, chunks: list[Chunk]) -> list[dict[str, int]]:
+        return self.model.rank(query, [chunk.get_raw_text() for chunk in chunks])
+
+
+# TODO: add models parameters to global config file
+# TODO: add exception handling when response have more tokens than was set
+# TODO: find a way to restrict the model for providing too long answers
+
+
+class LocalLLM:
+    def __init__(self):
+        self.model = AutoModelForCausalLM.from_pretrained(
+            **settings.local_llm.model_dump()
+        )
+
+    """
+    Produces the response to user's prompt
+
+    stream -> flag, determines weather we need to wait until the response is ready or can show it token by token
+
+    TODO: invent a way to really stream the answer (as return value)
+    """
+
+    def get_response(
+        self,
+        prompt: str,
+        stream: bool = True,
+        logging: bool = True,
+        use_default_config: bool = True,
+    ) -> str:
+
+        with open("../prompt.txt", "w") as f:
+            f.write(prompt)
+
+        generated_text = ""
+        tokenized_text: list[int] = self.model.tokenize(text=prompt)
+        response: list[int] = self.model.generate(
+            tokens=tokenized_text, **settings.local_llm.model_dump()
+        )
+
+        if logging:
+            print(response)
+
+        if not stream:
+            return self.model.detokenize(response)
+
+        for token in response:
+            chunk = self.model.detokenize([token])
+            generated_text += chunk
+            if logging:
+                print(chunk, end="", flush=True)  # flush -> clear the buffer
+
+        return generated_text
+
+
+class GeminiLLM:
+    def __init__(self, model="gemini-2.0-flash"):
+        self.client = genai.Client(api_key=settings.api_key)
+        self.model = model
+
+    def get_response(
+        self,
+        prompt: str,
+        stream: bool = True,
+        logging: bool = True,
+        use_default_config: bool = False,
+    ) -> str:
+        path_to_prompt = os.path.join(BASE_DIR, "prompt.txt")
+        with open(path_to_prompt, "w", encoding="utf-8", errors="replace") as f:
+            f.write(prompt)
+
+        response = self.client.models.generate_content(
+            model=self.model,
+            contents=prompt,
+            config=(
+                types.GenerateContentConfig(**settings.gemini_generation.model_dump())
+                if use_default_config
+                else None
+            ),
+        )
+
+        return response.text
+
+    async def get_streaming_response(
+        self,
+        prompt: str,
+        stream: bool = True,
+        logging: bool = True,
+        use_default_config: bool = False,
+    ):
+        path_to_prompt = os.path.join(BASE_DIR, "prompt.txt")
+        with open(path_to_prompt, "w", encoding="utf-8", errors="replace") as f:
+            f.write(prompt)
+
+        response = self.client.models.generate_content_stream(
+            model=self.model,
+            contents=prompt,
+            config=(
+                types.GenerateContentConfig(**settings.gemini_generation.model_dump())
+                if use_default_config
+                else None
+            ),
+        )
+
+        for chunk in response:
+            yield chunk
+
+
+class GeminiEmbed:
+    def __init__(self, model="text-embedding-004"):
+        self.client = genai.Client(api_key=settings.api_key)
+        self.model = model
+        self.settings = GeminiEmbeddingSettings()
+
+    def encode(self, text: str | list[str]) -> list[Tensor]:
+
+        if isinstance(text, str):
+            text = [text]
+
+        output: list[Tensor] = []
+        max_batch_size = 100  # can not be changed due to google restrictions
+
+        for i in range(0, len(text), max_batch_size):
+            batch = text[i : i + max_batch_size]
+            response = self.client.models.embed_content(
+                model=self.model,
+                contents=batch,
+                config=types.EmbedContentConfig(
+                    **settings.gemini_embedding.model_dump()
+                ),
+            ).embeddings
+
+            for i, emb in enumerate(response):
+                output.append(emb.values)
+
+        return output
+
+    def get_vector_dimensionality(self) -> int | None:
+        return getattr(self.settings, "output_dimensionality")
+
+
+class Wrapper:
+    def __init__(self, model: str = "gemini-2.0-flash"):
+        self.model = model
+        self.client = genai.Client(api_key=settings.api_key)
+
+    def wrap(self, prompt: str) -> str:
+        response = self.client.models.generate_content(
+            model=self.model,
+            contents=prompt,
+            config=types.GenerateContentConfig(**settings.gemini_wrapper.model_dump())
+        )
+
+        return response.text

app/core/processor.py

@@ -0,0 +1,284 @@
+from langchain_community.document_loaders import (
+    PyPDFLoader,
+    UnstructuredWordDocumentLoader,
+    TextLoader,
+    CSVLoader,
+    UnstructuredMarkdownLoader,
+)
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+from app.core.models import Embedder
+from app.core.chunks import Chunk
+import nltk  # used for proper tokenizer workflow
+from uuid import (
+    uuid4,
+)  # for generating unique id as hex (uuid4 is used as it generates ids form pseudo random numbers unlike uuid1 and others)
+import numpy as np
+from app.settings import logging, settings
+
+
+# TODO: replace PDFloader since it is completely unusable OR try to fix it
+
+
+class DocumentProcessor:
+    """
+    TODO: determine the most suitable chunk size
+
+    chunks -> the list of chunks from loaded files
+    chunks_unsaved -> the list of recently added chunks that have not been saved to db yet
+    processed -> the list of files that were already splitted into chunks
+    unprocessed -> !processed
+    text_splitter -> text splitting strategy
+    """
+
+    def __init__(self, embedder: Embedder):
+        self.chunks: list[Chunk] = []
+        self.chunks_unsaved: list[Chunk] = []
+        self.processed: list[Document] = []
+        self.unprocessed: list[Document] = []
+        self.embedder = embedder
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            **settings.text_splitter.model_dump()
+        )
+
+    """
+    Measures cosine between two vectors
+    """
+
+    def cosine_similarity(self, vec1, vec2):
+        return vec1 @ vec2 / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
+
+    """
+    Updates a list of the most relevant chunks without interacting with db
+    """
+
+    def update_most_relevant_chunk(
+        self,
+        chunk: list[np.float64, Chunk],
+        relevant_chunks: list[list[np.float64, Chunk]],
+        mx_len=15,
+    ):
+        relevant_chunks.append(chunk)
+        for i in range(len(relevant_chunks) - 1, 0, -1):
+            if relevant_chunks[i][0] > relevant_chunks[i - 1][0]:
+                relevant_chunks[i], relevant_chunks[i - 1] = (
+                    relevant_chunks[i - 1],
+                    relevant_chunks[i],
+                )
+            else:
+                break
+
+        if len(relevant_chunks) > mx_len:
+            del relevant_chunks[-1]
+
+    """
+    Loads one file - extracts text from file
+
+    TODO: Replace UnstructuredWordDocumentLoader with Docx2txtLoader
+    TODO: Play with .pdf and text from img extraction
+    TODO: Try chunking with llm
+
+    add_to_unprocessed -> used to add loaded file to the list of unprocessed(unchunked) files if true
+    """
+
+    def load_document(
+        self, filepath: str, add_to_unprocessed: bool = False
+    ) -> list[Document]:
+        loader = None
+
+        if filepath.endswith(".pdf"):
+            loader = PyPDFLoader(
+                file_path=filepath
+            )  # splits each presentation into slides and processes it as separate file
+        elif filepath.endswith(".docx") or filepath.endswith(".doc"):
+            # loader = Docx2txtLoader(file_path=filepath) ## try it later, since UnstructuredWordDocumentLoader is extremly slow
+            loader = UnstructuredWordDocumentLoader(file_path=filepath)
+        elif filepath.endswith(".txt"):
+            loader = TextLoader(file_path=filepath)
+        elif filepath.endswith(".csv"):
+            loader = CSVLoader(file_path=filepath)
+        elif filepath.endswith(".json"):
+            loader = TextLoader(file_path=filepath)
+        elif filepath.endswith(".md"):
+            loader = UnstructuredMarkdownLoader(file_path=filepath)
+
+        if loader is None:
+            raise RuntimeError("Unsupported type of file")
+
+        documents: list[Document] = (
+            []
+        )  # We can not assign a single value to the document since .pdf are splitted into several files
+        try:
+            documents = loader.load()
+            # print("-" * 100, documents, "-" * 100, sep="\n")
+        except Exception:
+            raise RuntimeError("File is corrupted")
+
+        if add_to_unprocessed:
+            for doc in documents:
+                self.unprocessed.append(doc)
+
+        return documents
+
+    """
+    Similar to load_document, but for multiple files
+
+    add_to_unprocessed -> used to add loaded files to the list of unprocessed(unchunked) files if true
+    """
+
+    def load_documents(
+        self, documents: list[str], add_to_unprocessed: bool = False
+    ) -> list[Document]:
+        extracted_documents: list[Document] = []
+
+        for doc in documents:
+            temp_storage: list[Document] = []
+
+            try:
+                temp_storage = self.load_document(
+                    filepath=doc, add_to_unprocessed=False
+                )  # In some cases it should be True, but i can not imagine any :(
+            except Exception as e:
+                logging.error(
+                    "Error at load_documents while loading %s", doc, exc_info=e
+                )
+                continue
+
+            for extrc_doc in temp_storage:
+                extracted_documents.append(extrc_doc)
+
+                if add_to_unprocessed:
+                    self.unprocessed.append(extrc_doc)
+
+        return extracted_documents
+
+    """
+    Generates chunks with recursive splitter from the list of unprocessed files, add files to the list of processed, and clears unprocessed
+
+    TODO: try to split text with other llm (not really needed, but we should at least try it)
+    """
+
+    def generate_chunks(self, query: str = "", embedding: bool = False):
+        most_relevant = []
+
+        if embedding:
+            query_embedded = self.embedder.encode(query)
+
+        for document in self.unprocessed:
+            self.processed.append(document)
+
+            text: list[str] = self.text_splitter.split_documents([document])
+            lines: list[str] = document.page_content.split("\n")
+
+            for chunk in text:
+                start_l, end_l = self.get_start_end_lines(
+                    splitted_text=lines,
+                    start_char=chunk.metadata.get("start_index", 0),
+                    end_char=chunk.metadata.get("start_index", 0)
+                    + len(chunk.page_content),
+                )
+
+                newChunk = Chunk(
+                    id=uuid4(),
+                    filename=document.metadata.get("source", ""),
+                    page_number=document.metadata.get("page", 0),
+                    start_index=chunk.metadata.get("start_index", 0),
+                    start_line=start_l,
+                    end_line=end_l,
+                    text=chunk.page_content,
+                )
+
+                if embedding:
+                    chunk_embedded = self.embedder.encode(newChunk.text)
+                    similarity = self.cosine_similarity(query_embedded, chunk_embedded)
+                    self.update_most_relevant_chunk(
+                        [similarity, newChunk], most_relevant
+                    )
+
+                self.chunks.append(newChunk)
+                self.chunks_unsaved.append(newChunk)
+
+            self.unprocessed = []
+        return most_relevant
+
+    """
+    Determines the line, were the chunk starts and ends (1-based indexing)
+
+    Some magic stuff here. To be honest, i understood it after 7th attempt
+
+    TODO: invent more efficient way
+
+    splitted_text -> original text splitted by \n
+    start_char -> index of symbol, were current chunk starts
+    end_char ->  index of symbol, were current chunk ends
+    debug_mode -> flag, which enables printing useful info about the process
+    """
+
+    def get_start_end_lines(
+        self,
+        splitted_text: list[str],
+        start_char: int,
+        end_char: int,
+        debug_mode: bool = False,
+    ) -> tuple[int, int]:
+        if debug_mode:
+            logging.info(splitted_text)
+
+        start, end, char_ct = 0, 0, 0
+        iter_count = 1
+
+        for i, line in enumerate(splitted_text):
+            if debug_mode:
+                logging.info(
+                    f"start={start_char}, current={char_ct}, end_current={char_ct + len(line) + 1}, end={end_char}, len={len(line)}, iter={iter_count}\n"
+                )
+
+            if char_ct <= start_char <= char_ct + len(line) + 1:
+                start = i + 1
+            if char_ct <= end_char <= char_ct + len(line) + 1:
+                end = i + 1
+                break
+
+            iter_count += 1
+            char_ct += len(line) + 1
+
+        if debug_mode:
+            logging.info(f"result => {start} {end}\n\n\n")
+
+        return start, end
+
+    """
+    Note: it should be used only once to download tokenizers, futher usage is not recommended
+    """
+
+    def update_nltk(self) -> None:
+        nltk.download("punkt")
+        nltk.download("averaged_perceptron_tagger")
+
+    """
+    For now the system works as follows: we save recently loaded chunks in two arrays:
+        chunks - for all chunks, even for that ones that havn't been saveed to db
+        chunks_unsaved - for chunks that have been added recently
+    I do not know weather we really need to store all chunks that were added in the
+    current session, but chunks_unsaved are used to avoid dublications while saving to db.
+    """
+
+    def clear_unsaved_chunks(self):
+        self.chunks_unsaved = []
+
+    def get_all_chunks(self) -> list[Chunk]:
+        return self.chunks
+
+    """
+    If we want to save chunks to db, we need to clear the temp storage to avoid dublications
+    """
+
+    def get_and_save_unsaved_chunks(self) -> list[Chunk]:
+        chunks_copy: list[Chunk] = self.chunks.copy()
+        self.clear_unsaved_chunks()
+        return chunks_copy
+
+
+if __name__ == "__main__":
+    document = DocumentProcessor()
+    print(document.__getattribute__())

app/core/rag_generator.py

@@ -0,0 +1,173 @@
+from typing import Any, AsyncGenerator
+from app.core.models import LocalLLM, Embedder, Reranker, GeminiLLM, GeminiEmbed, Wrapper
+from app.core.processor import DocumentProcessor
+from app.core.database import VectorDatabase
+import time
+import os
+from app.settings import settings, BASE_DIR
+
+
+class RagSystem:
+    def __init__(self):
+        self.embedder = (
+            GeminiEmbed()
+            if settings.use_gemini
+            else Embedder(model=settings.models.embedder_model)
+        )
+        self.reranker = Reranker(model=settings.models.reranker_model)
+        self.processor = DocumentProcessor(self.embedder)
+        self.db = VectorDatabase(embedder=self.embedder)
+        self.llm = GeminiLLM() if settings.use_gemini else LocalLLM()
+        self.wrapper = Wrapper()
+
+    """
+    Provides a prompt with substituted context from chunks
+
+    TODO: add template to prompt without docs
+    """
+
+    def get_general_prompt(self, user_prompt: str, collection_name: str) -> str:
+        enhanced_prompt = self.enhance_prompt(user_prompt.strip())
+
+        relevant_chunks = self.db.search(collection_name, query=enhanced_prompt, top_k=30)
+        if relevant_chunks is not None and len(relevant_chunks) > 0:
+            ranks = self.reranker.rank(query=enhanced_prompt, chunks=relevant_chunks)[
+                : min(5, len(relevant_chunks))
+            ]
+            relevant_chunks = [relevant_chunks[rank["corpus_id"]] for rank in ranks]
+        else:
+            relevant_chunks = []
+
+        sources = ""
+        prompt = ""
+
+        for chunk in relevant_chunks:
+            citation = (
+                f"[Source: {chunk.filename}, "
+                f"Page: {chunk.page_number}, "
+                f"Lines: {chunk.start_line}-{chunk.end_line}, "
+                f"Start: {chunk.start_index}]\n\n"
+            )
+            sources += f"Original text:\n{chunk.get_raw_text()}\nCitation:{citation}"
+
+        with open(
+            os.path.join(BASE_DIR, "app", "prompt_templates", "test2.txt")
+        ) as prompt_file:
+            prompt = prompt_file.read()
+
+        prompt += (
+            "**QUESTION**: "
+            f"{enhanced_prompt}\n"
+            "**CONTEXT DOCUMENTS**:\n"
+            f"{sources}\n"
+        )
+        print(prompt)
+        return prompt
+
+    def enhance_prompt(self, original_prompt: str) -> str:
+        path_to_wrapping_prompt = os.path.join(BASE_DIR, "app", "prompt_templates", "wrapper.txt")
+        enhanced_prompt = ""
+        with open(path_to_wrapping_prompt, "r") as f:
+            enhanced_prompt = f.read().replace("[USERS_PROMPT]", original_prompt)
+        return self.wrapper.wrap(enhanced_prompt)
+
+    """
+    Splits the list of documents into groups with 'split_by' docs (done to avoid qdrant_client connection error handling), loads them,
+    splits into chunks, and saves to db
+    """
+
+    def upload_documents(
+        self,
+        collection_name: str,
+        documents: list[str],
+        split_by: int = 3,
+        debug_mode: bool = True,
+    ) -> None:
+
+        for i in range(0, len(documents), split_by):
+
+            if debug_mode:
+                print(
+                    "<"
+                    + "-" * 10
+                    + "New document group is taken into processing"
+                    + "-" * 10
+                    + ">"
+                )
+
+            docs = documents[i : i + split_by]
+
+            loading_time = 0
+            chunk_generating_time = 0
+            db_saving_time = 0
+
+            print("Start loading the documents")
+            start = time.time()
+            self.processor.load_documents(documents=docs, add_to_unprocessed=True)
+            loading_time = time.time() - start
+
+            print("Start loading chunk generation")
+            start = time.time()
+            self.processor.generate_chunks()
+            chunk_generating_time = time.time() - start
+
+            print("Start saving to db")
+            start = time.time()
+            self.db.store(collection_name, self.processor.get_and_save_unsaved_chunks())
+            db_saving_time = time.time() - start
+
+            if debug_mode:
+                print(
+                    f"loading time = {loading_time}, chunk generation time = {chunk_generating_time}, saving time = {db_saving_time}\n"
+                )
+
+    def extract_text(self, response) -> str:
+        text = ""
+        try:
+            text = response.candidates[0].content.parts[0].text
+        except Exception as e:
+            print(e)
+        return text
+
+    """
+    Produces answer to user's request. First, finds the most relevant chunks, generates prompt with them, and asks llm
+    """
+
+    async def generate_response(
+        self, collection_name: str, user_prompt: str, stream: bool = True
+    ) -> str:
+        general_prompt = self.get_general_prompt(
+            user_prompt=user_prompt, collection_name=collection_name
+        )
+
+        return self.llm.get_response(prompt=general_prompt)
+
+    async def generate_response_stream(
+        self, collection_name: str, user_prompt: str, stream: bool = True
+    ) -> AsyncGenerator[Any, Any]:
+        general_prompt = self.get_general_prompt(
+            user_prompt=user_prompt, collection_name=collection_name
+        )
+
+        async for chunk in self.llm.get_streaming_response(
+            prompt=general_prompt, stream=True
+        ):
+            yield self.extract_text(chunk)
+
+    """
+    Produces the list of the most relevant chunkВs
+    """
+
+    def get_relevant_chunks(self, collection_name: str, query):
+        relevant_chunks = self.db.search(collection_name, query=query, top_k=15)
+        relevant_chunks = [
+            relevant_chunks[ranked["corpus_id"]]
+            for ranked in self.reranker.rank(query=query, chunks=relevant_chunks)
+        ]
+        return relevant_chunks
+
+    def create_new_collection(self, collection_name: str) -> None:
+        self.db.create_collection(collection_name)
+
+    def get_collections_names(self) -> list[str]:
+        return self.db.get_collections()

app/core/response_parser.py

@@ -0,0 +1,29 @@
+from app.core.document_validator import path_is_valid
+import re
+
+"""
+Replaces the matched regular exp with link via html <a></a>
+"""
+
+
+def create_url(match: re.Match) -> str:
+    path: str = match.group(1)
+    page: str = match.group(2)
+    lines: str = match.group(3)
+    start: str = match.group(4)
+
+    if not path_is_valid(path):
+        return ""
+
+    return f'<a href="/viewer?path={path}&page={page}&lines={lines}&start={start}">[Source]</a>'
+
+
+"""
+Replaces all occurrences of citation pattern with links
+"""
+
+
+def add_links(response: str) -> str:
+
+    citation_format = r"\[Source:\s*([^,]+?)\s*,\s*Page:\s*(\d+)\s*,\s*Lines:\s*(\d+\s*-\s*\d+)\s*,\s*Start:?\s*(\d+)\]"
+    return re.sub(pattern=citation_format, repl=create_url, string=response)

app/core/utils.py

@@ -0,0 +1,200 @@
+from fastapi.templating import Jinja2Templates
+from fastapi import Request, UploadFile
+
+from app.backend.controllers.chats import list_user_chats, verify_ownership_rights
+from app.backend.controllers.users import get_current_user
+from app.backend.models.users import User
+from app.core.rag_generator import RagSystem
+from app.settings import BASE_DIR
+
+from uuid import uuid4
+import markdown
+import os
+
+rag = None
+
+
+# <----------------------- System ----------------------->
+def initialize_rag() -> RagSystem:
+    global rag
+    if rag is None:
+        rag = RagSystem()
+    return rag
+
+
+# <----------------------- Tools ----------------------->
+"""
+Updates response context and adds context of navbar (role, instance(or none)) and footer (none)
+"""
+
+
+def extend_context(context: dict, selected: int = None):
+    user = get_current_user(context.get("request"))
+    navbar = {
+        "navbar": False,
+        "navbar_path": "components/navbar.html",
+        "navbar_context": {
+            "chats": [],
+            "user": {"role": "user" if user else "guest", "instance": user},
+        },
+    }
+    sidebar = {
+        "sidebar": True,
+        "sidebar_path": "components/sidebar.html",
+        "sidebar_context": {
+            "selected": selected if selected is not None else None,
+            "chat_groups": list_user_chats(user.id) if user else [],
+        },
+    }
+    footer = {"footer": False, "footer_context": None}
+
+    context.update(**navbar)
+    context.update(**footer)
+    context.update(**sidebar)
+
+    return context
+
+
+"""
+Validates chat viewing permission by comparing user's chats and requested one
+"""
+
+
+def protect_chat(user: User, chat_id: int) -> bool:
+    return verify_ownership_rights(user, chat_id)
+
+
+async def save_documents(
+    collection_name: str,
+    files: list[UploadFile],
+    RAG: RagSystem,
+    user: User,
+    chat_id: int,
+) -> None:
+    storage = os.path.join(
+        BASE_DIR,
+        "chats_storage",
+        f"user_id={user.id}",
+        f"chat_id={chat_id}",
+        "documents",
+    )
+    docs = []
+
+    if files is None or len(files) == 0:
+        return
+
+    os.makedirs(os.path.join(storage, "pdfs"), exist_ok=True)
+
+    for file in files:
+        content = await file.read()
+
+        if file.filename.endswith(".pdf"):
+            saved_file = os.path.join(storage, "pdfs", str(uuid4()) + ".pdf")
+        else:
+            saved_file = os.path.join(
+                storage, str(uuid4()) + "." + file.filename.split(".")[-1]
+            )
+
+        with open(saved_file, "wb") as f:
+            f.write(content)
+
+        docs.append(saved_file)
+
+    if len(files) > 0:
+        RAG.upload_documents(collection_name, docs)
+
+
+def get_pdf_path(path: str) -> str:
+    parts = path.split("chats_storage")
+    if len(parts) < 2:
+        return ""
+    return "chats_storage" + "".join(parts[1:])
+
+
+def construct_collection_name(user: User, chat_id: int) -> str:
+    return f"user_id_{user.id}_chat_id_{chat_id}"
+
+
+def create_collection(user: User, chat_id: int, RAG: RagSystem) -> None:
+    if RAG is None:
+        raise RuntimeError("RAG was not initialized")
+
+    RAG.create_new_collection(construct_collection_name(user, chat_id))
+    print(rag.get_collections_names())
+
+
+def lines_to_markdown(lines: list[str]) -> list[str]:
+    return [markdown.markdown(line) for line in lines]
+
+
+# <----------------------- Handlers ----------------------->
+def PDFHandler(
+    request: Request, path: str, page: int, templates
+) -> Jinja2Templates.TemplateResponse:
+    print(path)
+    url_path = get_pdf_path(path=path)
+    print(url_path)
+
+    current_template = "pages/show_pdf.html"
+    return templates.TemplateResponse(
+        current_template,
+        extend_context(
+            {
+                "request": request,
+                "page": str(page or 1),
+                "url_path": url_path,
+                "user": get_current_user(request),
+            }
+        ),
+    )
+
+
+def TextHandler(
+    request: Request, path: str, lines: str, templates
+) -> Jinja2Templates.TemplateResponse:
+    file_content = ""
+    with open(path, "r") as f:
+        file_content = f.read()
+
+    start_line, end_line = map(int, lines.split("-"))
+
+    text_before_citation = []
+    text_after_citation = []
+    citation = []
+    anchor_added = False
+
+    for index, line in enumerate(file_content.split("\n")):
+        if line == "" or line == "\n":
+            continue
+        if index + 1 < start_line:
+            text_before_citation.append(line)
+        elif end_line < index + 1:
+            text_after_citation.append(line)
+        else:
+            anchor_added = True
+            citation.append(line)
+
+    current_template = "pages/show_text.html"
+
+    return templates.TemplateResponse(
+        current_template,
+        extend_context(
+            {
+                "request": request,
+                "text_before_citation": lines_to_markdown(text_before_citation),
+                "text_after_citation": lines_to_markdown(text_after_citation),
+                "citation": lines_to_markdown(citation),
+                "anchor_added": anchor_added,
+                "user": get_current_user(request),
+            }
+        ),
+    )
+
+
+"""
+Optional handler
+"""
+
+
+def DocHandler():
+    pass

app/email_templates/password_reset.html

@@ -0,0 +1,80 @@
+<html lang="en">
+<head>
+    <style>
+        body {{
+            font-family: Arial, sans-serif;
+            line-height: 1.6;
+            color: #333333;
+            margin: 0;
+            padding: 0;
+            background-color: #f4f4f4;
+        }}
+
+        .container {{
+            max-width: 600px;
+            margin: 20px auto;
+            padding: 20px;
+            background: #ffffff;
+            border-radius: 8px;
+            box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
+        }}
+
+        .header {{
+            background-color: #007bff;
+            color: #ffffff;
+            padding: 10px 20px;
+            border-top-left-radius: 8px;
+            border-top-right-radius: 8px;
+            text-align: center;
+        }}
+
+        .content {{
+            padding: 20px;
+        }}
+
+        .button {{
+            display: inline-block;
+            background-color: #28a745;
+            color: #ffffff;
+            padding: 10px 20px;
+            border-radius: 5px;
+            text-decoration: none;
+            margin-top: 15px;
+        }}
+
+        .footer {{
+            text-align: center;
+            margin-top: 20px;
+            font-size: 1.2em;
+            color: #777777;
+        }}
+
+        p {{
+            margin-bottom: 15px;
+        }}
+    </style>
+</head>
+<body>
+<div class="container">
+    <div class="header">
+        <h2>Password Reset Request</h2>
+    </div>
+    <div class="content">
+        <p>Hello,</p>
+        <p>We received a request to reset your password for your account. If you initiated this request, please click on
+            the link below to set a new password:</p>
+        <p style="text-align: center;">
+            <a href="{application_server_url}?token={reset_token}" class="button">Reset Your Password</a>
+        </p>
+        <p>This password reset link is valid for <b>{expires_in} minutes</b>. For security
+            reasons, if you do not reset your password within this timeframe, you will need to submit another
+            request.</p>
+        <p>If you did not request a password reset, please ignore this email. Your password will remain unchanged.</p>
+        <p>Thanks,</p>
+    </div>
+    <div class="footer">
+        <p>The Ultimate RAG Team</p>
+    </div>
+</div>
+</body>
+</html>

app/frontend/static/styles.css

@@ -0,0 +1,377 @@
+#pdf-container {
+    margin: 0 auto;
+    max-width: 100%;
+    overflow-x: auto;
+    text-align: center;
+    padding: 20px 0;
+}
+
+#pdf-canvas {
+    margin: 0 auto;
+    display: block;
+    max-width: 100%;
+    box-shadow: 0 0 5px rgba(0,0,0,0.2);
+}
+
+#pageNum {
+    height: 40px; /* optional */
+    font-size: 16px; /* makes text inside input larger */
+    padding: 10px;
+    width: 9vh;    /* optional for more padding inside the box */
+}
+
+.page-input {
+    width: 60px;
+    padding: 8px;
+    padding-right: 40px; /* reserve space for label inside input box */
+    text-align: center;
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    -moz-appearance: textfield;
+}
+
+.page-input-label {
+    position: absolute;
+    right: 12px;
+    top: 50%;
+    transform: translateY(-50%);
+    font-size: 12px;
+    color: #666;
+    pointer-events: none;
+    background-color: #fff; /* Match background to prevent text overlapping */
+    padding-left: 4px;
+}
+
+.page-input-container {
+    position: relative;
+    display: inline-flex;
+    align-items: center;
+}
+
+/* Hide number arrows in Chrome/Safari */
+.page-input::-webkit-outer-spin-button,
+.page-input::-webkit-inner-spin-button {
+    -webkit-appearance: none;
+    margin: 0;
+}
+
+/* Pagination styling */
+.pagination-container {
+    margin: 20px 0;
+    text-align: center;
+}
+
+.pagination {
+    display: inline-flex;
+    align-items: center;
+}
+
+.pagination-button {
+    padding: 8px 16px;
+    background: #4a6fa5;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    gap: 5px;
+}
+
+.pagination-button-text:hover {
+    background-color: #e0e0e0;
+    transform: translateY(-1px);
+}
+
+.pagination-button-text:active {
+    transform: translateY(0);
+}
+
+.text-viewer {
+    overflow-y: auto;  /* Enables vertical scrolling when needed */
+    height: 100%; 
+    width: 100%; /* Or whatever height you prefer */
+    font-family: monospace;
+    white-space: pre-wrap; /* Preserve line breaks but wrap text */
+    background: #f8f8f8;
+    padding: 20px;
+    border-radius: 5px;
+    line-height: 1.5;
+}
+
+.citation {
+    background-color: rgba(0, 255, 0, 0.2);
+    padding: 2px 0;
+}
+
+.no-content {
+    color: #999;
+    font-style: italic;
+}
+
+.pagination-container-text {
+    margin: 20px 0;
+    text-align: center;
+}
+
+.pagination-button-text {
+    padding: 8px 16px;
+    background: #4a6fa5;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+}
+
+
+
+/*           --------------------------------------------               */
+
+body {
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+    background-color: #f7f7f8;
+    color: #111827;
+    margin: 0;
+    overflow: hidden; 
+    height: 100vh;
+    padding: 0;
+    display: flex;
+}
+
+.sidebar {
+    width: 260px;
+    height: 100vh;
+    background-color: #1F2937;
+    /* border-right: 1px solid #e1e4e8; */
+    overflow-y: auto; 
+    padding: 8px;
+    position: sticky;
+    top: 0;
+}
+
+.chat-page {
+    background-color: #111827;
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    height: 100vh;
+    overflow: hidden; /* Prevent double scrollbars */
+}
+
+.container {
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    padding: 0;
+    max-width: 100%;
+    height: 100%;
+}
+
+/* Chat messages section */
+.chat-messages {
+    flex: 1;
+    overflow-y: auto; /* Make only this section scrollable */
+    padding: 16px;
+    display: flex;
+    flex-direction: column;
+    gap: 16px;
+}
+
+/* Input area - stays fixed at bottom */
+.input-group {
+    /* padding: 16px;
+    background-color: #44444C; */
+    /* border-top: 1px solid #e1e4e8; */
+    position: sticky;
+    bottom: 0;
+}
+
+/* General styles */
+
+/* Sidebar styles */
+
+.chat-group {
+    font-weight: 500;
+    color: #9bb8d3;
+    text-transform: uppercase;
+    letter-spacing: 0.5px;
+    font-size: 12px;
+    padding: 8px 12px;
+}
+
+.btn {
+    border-radius: 10px;
+    padding: 8px 12px;
+    font-size: 14px;
+    transition: all 0.2s;
+}
+
+.btn-success {
+    background-color: #19c37d;
+    border-color: #19c37d;
+}
+
+.btn-success:hover {
+    background-color: #16a369;
+    border-color: #16a369;
+}
+
+.btn-outline-secondary {
+    /* border-color: #e1e4e8; */
+    color: #374151;
+    background-color: transparent;
+}
+
+.btn-outline-secondary:hover {
+    background-color: #273c50;
+    border-color: #e1e4e8;
+}
+
+.btn-outline-light {
+    border-color: #e1e4e8;
+    color: #666;
+    background-color: transparent;
+}
+
+.btn-outline-light:hover {
+    background-color: #e9ecef;
+    border-color: #e1e4e8;
+}
+
+/* Chat page styles */
+
+.message {
+    max-width: 80%;
+    padding: 12px 16px;
+    border-radius: 12px;
+    line-height: 1.5;
+}
+
+.user-message {
+    align-self: flex-end;
+    background-color: #19c37d;
+    color: white;
+    border-bottom-right-radius: 4px;
+}
+
+.assistant-message {
+    align-self: flex-start;
+    background-color: #f0f4f8;
+    border-bottom-left-radius: 4px;
+}
+
+.message-header {
+    font-weight: 600;
+    font-size: 12px;
+    margin-bottom: 4px;
+    color: #666;
+}
+
+.user-message .message-header {
+    color: rgba(255, 255, 255, 0.8);
+}
+
+.message-content {
+    font-size: 14px;
+}
+
+
+.form-control {
+    border-radius: 6px;
+    padding: 10px 12px;
+    background-color: #374151;
+    /* border: 1px solid #e1e4e8; */
+}
+
+.form-control:focus {
+    box-shadow: none;
+    border-color: #19c37d;
+}
+
+/* File input button */
+.btn-outline-secondary {
+    position: relative;
+}
+
+.btn-outline-secondary input[type="file"] {
+    position: absolute;
+    opacity: 0;
+    width: 100%;
+    height: 100%;
+    top: 0;
+    left: 0;
+    cursor: pointer;
+}
+
+/* Scrollbar styles */
+::-webkit-scrollbar {
+    width: 8px;
+}
+
+::-webkit-scrollbar-track {
+    background: #f1f1f1;
+}
+
+::-webkit-scrollbar-thumb {
+    background: #ccc;
+    border-radius: 4px;
+}
+
+::-webkit-scrollbar-thumb:hover {
+    background: #aaa;
+}
+
+/* Responsive adjustments */
+@media (max-width: 768px) {
+    .sidebar {
+        width: 220px;
+    }
+    
+    .message {
+        max-width: 90%;
+    }
+}
+
+#queryInput {
+    background-color: #374151;
+    color: white;
+}
+
+#queryInput:focus {
+    background-color: #374151;
+    color: white;
+    outline: none;
+    box-shadow: none;
+    border-color: #19c37d; /* optional green border for focus, remove if unwanted */
+}
+
+#searchButton {
+    background-color: #374151;
+}
+
+#fileInput {
+    background-color: #374151;
+}
+
+
+/* For the placeholder text color */
+#queryInput::placeholder {
+    color: rgba(255, 255, 255, 0.7); /* Slightly transparent white */
+}
+
+.auth-card {
+    background-color: #1F2937;
+    border: none;
+    border-radius: 12px;
+}
+
+.auth-input {
+    background-color: #374151 !important;
+    border: none !important;
+    color: white !important;
+}
+
+.auth-input-group-text {
+    background-color: #374151 !important;
+    border: none !important;
+}

app/frontend/templates/base.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {% block title %}
+    {% endblock %}
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.5/font/bootstrap-icons.css" rel="stylesheet">
+    <link href="/static/styles.css" rel="stylesheet">
+    {% block head_scripts %}
+    {% endblock %}
+</head>
+<body>
+    {% if navbar %}
+        {% with context=navbar_context %}
+            {% include navbar_path %}
+        {% endwith %}
+    {% endif %}
+
+    {% if sidebar %}
+        {% with context=sidebar_context %}
+            {% include sidebar_path %}
+        {% endwith %}
+    {% endif %}
+
+    {% block content %}
+        {% with context=sidebar_context %}
+            {% include sidebar_path %}
+        {% endwith %}
+    {% endblock %}
+
+    {% if footer %}
+        {% with context=footer_context %}
+            {% include footer_path %}
+        {% endwith %}
+    {% endif %}
+
+    {% block body_scripts %}
+    {% endblock %}
+</body>
+</html>

app/frontend/templates/components/navbar.html

@@ -0,0 +1,33 @@
+<!-- All the data is accessible via context  -->
+<div>
+    {% if context.user.role == "guest" %}
+        <p>Hello, guest!</p>
+    {% else %}
+        <p>Hello, {{ context.user.instance.email }}</p>
+    {% endif %}
+
+    <p>Today</p>
+    <ul>
+        {% for chat in context.chats.today %}
+            <li>{{ chat.title }}</li>
+        {% endfor %}
+    </ul>
+    <p>Last week</p>
+    <ul>
+        {% for chat in context.chats.last_week %}
+            <li>{{ chat.title }}</li>
+        {% endfor %}
+    </ul>
+    <p>Last month</p>
+    <ul>
+        {% for chat in context.chats.last_month %}
+            <li>{{ chat.title }}</li>
+        {% endfor %}
+    </ul>
+    <p>Later</p>
+     <ul>
+        {% for chat in context.chats.other %}
+            <li>{{ chat.title }}</li>
+        {% endfor %}
+    </ul>
+</div>

app/frontend/templates/components/sidebar.html

@@ -0,0 +1,26 @@
+<div class="sidebar">
+    <div class="d-flex justify-content-between align-items-center p-3">
+        <form action="/new_chat" method="post">
+            <button type="submit" class="btn btn-success w-100">+ Add new chat</button>
+        </form>
+    </div>
+
+    {% if context.chat_groups %}
+        {% for group in context.chat_groups %}
+            <div class="chat-group px-3 text mt-3">{{ group.title }}</div>
+            {% for chat in group.chats %}
+                <form action="/chats/id={{ chat.id }}" method="get" class="px-3 my-1">
+                    {% if context.selected == chat.id %}
+                        <button type="submit" class="btn btn-outline-secondary w-100 text-start text-truncate text-success">
+                            {{ chat.title }}
+                        </button>
+                    {% else %}
+                        <button type="submit" class="btn btn-outline-secondary w-100 text-start text-truncate text-white">
+                            {{ chat.title }}
+                        </button>
+                    {% endif %}
+                </form>
+            {% endfor %}
+        {% endfor %}
+    {% endif %}
+</div>

app/frontend/templates/pages/chat.html

@@ -0,0 +1,163 @@
+{% extends "base.html" %}
+
+{% block title %}
+    <title>
+        The Ultimate RAG
+    </title>
+{% endblock %}
+
+{% block content %}
+ <div class="chat-page">
+        <div class="container py-4">
+            <div id="chat-messages" class="chat-messages">
+                <!-- {% for message in history %}
+                    <div class="message {{ message.role }}-message">
+                        <div class="message-header">
+                            {{ "You" if message.role == "user" else "Assistant" }}
+                        </div>
+                        <div class="message-content">{{ message.content | safe }}</div>
+                    </div>
+                {% endfor %} -->
+            </div>
+
+            <form id="chat-form" class="input-group mt-4" enctype="multipart/form-data">
+                <input type="text" class="form-control" name="prompt" placeholder="Ask your question here" id="queryInput">
+                <label class="btn btn-outline-secondary btn-primary">
+                    📎<input type="file" id="fileInput" name="files" multiple hidden>
+                </label>
+                <button type="button" class="btn text-white" id="searchButton">Send</button>
+            </form>
+        </div>
+    </div>
+{% endblock %}
+
+{% block body_scripts %}
+    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
+    <script>
+        const initialChatId = "{{ chat_id }}";
+        const initialHistory = {{ history | tojson | safe }};
+        // Conversation state
+        let conversationId = initialChatId || null;
+
+        if (initialHistory && Array.isArray(initialHistory)) {
+            initialHistory.forEach(msg => {
+                addMessageToChat(msg.role, msg.content);
+            });
+        }
+
+        // Main chat function
+        document.getElementById('searchButton').addEventListener('click', async function() {
+            const query = document.getElementById('queryInput').value.trim();
+            if (!query) return alert('Please enter a question');
+
+            addMessageToChat('user', escapeHTML(query));
+            document.getElementById('queryInput').value = '';
+            const loadingId = addMessageToChat('assistant', '', true);
+
+            try {
+                const formData = new FormData();
+                const fileInput = document.getElementById('fileInput');
+                const files = fileInput.files;
+                for (let i = 0; i < files.length; i++) {
+                    formData.append('files', files[i]);
+                }
+                formData.append('prompt', query);
+                if (conversationId) formData.append('chat_id', conversationId);
+
+                const response = await fetch('/message_with_docs', {
+                    method: 'POST',
+                    body: formData
+                });
+
+                if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
+
+                const reader = response.body.getReader();
+                const decoder = new TextDecoder("utf-8");
+                let fullMessage = "";
+
+                while (true) {
+                    const { value, done } = await reader.read();
+                    if (done) break;
+
+                    const chunk = decoder.decode(value, { stream: true });
+                    fullMessage += chunk;
+                    updateMessageContent(loadingId, marked.parse(fullMessage));
+                }
+
+                removeMessage(loadingId);
+                const finalId = addMessageToChat('assistant', marked.parse(fullMessage));
+
+                try {
+                    const response = await fetch('/replace_message', {
+                        method: 'POST',
+                        headers: { "Content-Type": "application/json" },
+                        body: JSON.stringify({ message: fullMessage, chat_id: initialChatId })
+                    });
+
+                    if (!response.ok) throw new Error(`Replace error: ${response.status}`);
+
+                    const data = await response.json(); // expects { "updated_message": "..." }
+
+                    updateMessageContent(finalId, marked.parse(data.updated_message));
+
+                } catch (error) {
+                    console.error("Error replacing message:", error);
+                }
+
+            } catch (error) {
+                removeMessage(loadingId);
+                addMessageToChat('assistant', `Error: ${error.message}`, false, 'error');
+                console.error('Error:', error);
+            }
+        });
+
+        function updateMessageContent(messageId, newContent) {
+            const element = document.getElementById(messageId);
+            if (element) {
+                const contentDiv = element.querySelector('.message-content');
+                if (contentDiv) contentDiv.innerHTML = newContent;
+            }
+        }
+
+
+        // Message display helper
+        function addMessageToChat(role, content, isTemporary = false, className = '') {
+            const chatMessages = document.getElementById('chat-messages');
+            const messageId = 'msg-' + Date.now();
+
+            const messageDiv = document.createElement('div');
+            messageDiv.className = `message ${role}-message ${className}`;
+            messageDiv.id = messageId;
+
+            messageDiv.innerHTML = `
+                <div class="message-header">${role === 'user' ? 'You' : 'Assistant'}</div>
+                <div class="message-content">${marked.parse(content)}</div>
+            `;
+
+            chatMessages.appendChild(messageDiv);
+            chatMessages.scrollTop = chatMessages.scrollHeight;
+
+            return messageId; // always return the ID so you can update it later
+        }
+
+
+        function removeMessage(messageId) {
+            const element = document.getElementById(messageId);
+            if (element) element.remove();
+        }
+
+        function escapeHTML(str) {
+            const div = document.createElement('div');
+            div.textContent = str;
+            return div.innerHTML;
+        }
+        // New chat handler
+        document.querySelector('form[action="/new_chat"]').addEventListener('submit', function(e) {
+            e.preventDefault();
+            conversationId = null;
+            conversationHistory = [];
+            document.getElementById('chat-messages').innerHTML = '';
+            this.submit();
+        });
+    </script>
+{% endblock %}