Upload 24 files

agents/coordinator.py

@@ -0,0 +1,175 @@
+# agents/coordinator.py
+import logging
+import json
+import re
+from typing import Dict, Any, List
+from agents.nebius_simple import create_nebius_llm
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class CoordinatorAgent:
+    def __init__(self, nebius_api_key: str):
+        self.llm = create_nebius_llm(
+            api_key=nebius_api_key,
+            model="meta-llama/Llama-3.3-70B-Instruct-fast",
+            temperature=0.6  # Чуть выше для креативности при генерации историй
+        )
+
+    def _extract_json(self, text: str) -> dict:
+        """
+        Извлекает JSON из ответа LLM с улучшенной обработкой.
+        Поддерживает различные форматы ответов.
+        """
+        # 1. Убираем markdown code blocks
+        text = re.sub(r"```json\s*", "", text)
+        text = re.sub(r"```\s*", "", text)
+
+        # 2. Ищем JSON объект с помощью regex
+        json_pattern = r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}'
+        matches = re.findall(json_pattern, text, re.DOTALL)
+
+        if matches:
+        # Берём первый найденный JSON объект
+            json_str = matches[0].strip()
+        try:
+            return json.loads(json_str)
+        except json.JSONDecodeError as e:
+            logger.warning(f"Failed to parse extracted JSON: {e}")
+            logger.debug(f"JSON string was: {json_str}")
+
+        # 3. Fallback: пытаемся найти первые { и последние }
+        try:
+            start = text.index('{')
+            end = text.rindex('}') + 1
+            json_str = text[start:end].strip()
+            return json.loads(json_str)
+        except (ValueError, json.JSONDecodeError) as e:
+            logger.error(f"Could not extract JSON from response: {e}")
+            logger.debug(f"Response was: {text}")
+            raise
+
+
+    def analyze_input(self, user_text: str, attempt_count: int) -> Dict[str, Any]:
+        """
+        Анализ ввода пользователя.
+        Проверяет длину и является ли текст историей.
+        """
+        # 1. Проверка длины (менее 50 слов)
+        word_count = len(user_text.split())
+        if word_count < 50:
+            return {
+                "status": "insufficient",
+                "reason": "length",
+                "message": f"Your story is too short ({word_count} words). Please describe the plot in at "
+                           f"least 50 words so I can find the best match."
+            }
+
+        # 2. Проверка: это история или просто набор слов/вопрос?
+        check_prompt = f"""
+        Analyze if the following text is a narrative story/plot description or just random words/meta-talk.
+        Text: "{user_text}"
+
+        You MUST respond with ONLY valid JSON, nothing else. No explanations before or after.
+
+        Format:
+        {{"is_story": true/false, "reason": "brief explanation"}}
+        """
+        response = ""
+        try:
+            response = self.llm.complete(check_prompt).text
+            logger.debug(f"Story check response: {response[:200]}...")
+
+            # Очистка JSON
+            # cleaned_json = re.sub(r"```json|```", "", response).strip()
+            # analysis = json.loads(cleaned_json)
+            # ✅ Используем улучшенный парсинг
+            analysis = self._extract_json(response)
+
+
+            if not analysis.get("is_story", False):
+                return {
+                    "status": "insufficient",
+                    "reason": "not_story",
+                    "message": "This doesn't look like a story. Please describe "
+                               "a sequence of events, characters, and what happens to them."
+                }
+        except Exception as e:
+            logger.error(f"Coordinator story check error: {e}")
+            logger.debug(f"Full response: {response if 'response' in locals() else 'N/A'}")
+            # Fallback: пропускаем, если не удалось проверить
+            pass
+
+        # Если все ок
+        return {"status": "valid"}
+
+    def generate_suggestion(self, previous_inputs: List[str], genre: str) -> Dict[str, str]:
+        """
+        Генерирует историю за пользователя, если он не справляется.
+        ✅ С валидацией длины (минимум 50 слов)
+        """
+        # context = " ".join(previous_inputs)
+        context = " ".join(previous_inputs[-3:]) if previous_inputs else "nothing specific"
+
+        prompt = f"""
+    The user is trying to use a Movie Plot Search engine but fails to provide a good description.
+    
+    Based on their fragmented inputs: "{context}" (or generate something new if inputs are empty),
+    write a compelling, detailed movie plot summary in the {genre} genre.
+    
+    CRITICAL REQUIREMENTS:
+    - MINIMUM 60 words (aim for 70-90 words for a complete plot)
+    - Must include: main character(s), conflict, setting, stakes
+    - Clear narrative arc with beginning, middle, and potential resolution
+    - Engaging and specific details
+    - English language only
+    
+    Genre: {genre}
+    
+    Output ONLY the story text (60-90 words), no preamble or explanation:
+    """
+
+        story_text = self.llm.complete(prompt).text.strip()
+
+        # ✅ ВАЛИДАЦИЯ ДЛИНЫ сгенерированного текста
+        word_count = len(story_text.split())
+
+        if word_count < 50:
+            logger.warning(f"Generated {genre} plot too short ({word_count} words), expanding...")
+
+            # Повторная генерация с явным требованием расширения
+            expansion_prompt = f"""
+        The following {genre} plot is TOO SHORT ({word_count} words). 
+        Expand it to AT LEAST 60 words while keeping the same theme and characters.
+        Add more specific details about the conflict, character motivations, and stakes.
+
+        Current plot: {story_text}
+
+        Expanded version (60-90 words):
+        """
+
+            expansion_response = self.llm.complete(expansion_prompt)
+            story_text = expansion_response.text.strip()
+
+            # Проверка после расширения
+            final_word_count = len(story_text.split())
+            logger.info(f"Expanded plot to {final_word_count} words")
+        else:
+            logger.info(f"Generated {genre} plot has {word_count} words (valid)")
+
+        # Сообщения в зависимости от жанра
+        if genre == "romantic":
+            msg = ("I see you're having trouble. "
+                   " How about we search for a movie with a Romantic plot based on what you said?")
+        elif genre == "humorous":
+            msg = "Okay, maybe a Humorous story would be better?"
+        else:
+            msg = f"Let me suggest a {genre} plot for you."
+
+        return {
+            "status": "suggestion",
+            "genre": genre,
+            "message": msg,
+            "suggested_story": story_text
+        }

agents/critic_agent_nebius.py

@@ -0,0 +1,456 @@
+# agents/critic_agent_nebius.py
+from llama_index.core.agent import ReActAgent
+# from llama_index.llms.openai import OpenAI
+# from llama_index.llms.llama_api import LlamaAPI
+from llama_index.core.tools import FunctionTool
+
+from agents.nebius_simple import create_nebius_llm
+
+import datetime
+import re
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class FilmCriticAgent:
+    def __init__(self, nebius_api_key: str):
+        # ✅ Прямой Nebius LLM
+        self.llm = create_nebius_llm(
+            api_key=nebius_api_key,
+            model="meta-llama/Llama-3.3-70B-Instruct-fast",
+            # model="deepseek-ai/DeepSeek-R1-fast",
+            temperature=0.7
+        )
+
+        self.tools = [
+            self._create_overview_generation_tool(),
+            self._create_overview_refinement_tool(),
+            self._create_quality_assessment_tool()
+        ]
+
+        self.agent = ReActAgent.from_tools(
+            tools=self.tools,
+            llm=self.llm,
+            verbose=True,
+            max_iterations=15,
+            system_prompt=self._get_system_prompt()
+        )
+
+    @staticmethod
+    def _get_system_prompt() -> str:
+        """Статический метод для получения системного промпта.
+        Обновленный системный промпт без технических префиксов"""
+
+        return """You are a Film Critic Agent with expertise in movie analysis and synopsis writing.
+
+Your responsibilities:
+    1. Transform plot descriptions into professional movie overviews
+    2. Ensure overviews match the style and structure of real movie descriptions
+    3. Maintain narrative coherence and cinematic appeal
+    4. Refine overviews based on user feedback
+
+Use the Thought-Action-Observation cycle:
+    - Think about the narrative elements and cinematic potential
+    - Generate or refine the movie overview
+    - Assess the quality and completeness
+    - Make improvements until the overview meets professional standards
+    - your final response should contain only the clean overview text without any technical formatting.
+
+Write overviews in the style of IMDb or film database descriptions: 
+engaging, informative, and capturing the essence of the story.
+
+IMPORTANT OUTPUT RULES:
+    - Generate ONLY the overview text itself
+    - Do NOT add prefixes like "The final movie overview is:" or "Overview:"
+    - Do NOT add explanatory text or comments
+    - Write directly in the style of IMDb movie descriptions
+    - Use present tense and engaging language
+"""
+
+    # @staticmethod
+    def _create_overview_generation_tool(self) -> FunctionTool:
+        """Статический метод для создания инструмента генерации overview"""
+
+        def generate_movie_overview(plot_description: str) -> dict:
+            """Generate a professional movie overview from a plot description using LLM"""
+
+            # Placeholder для демонстрации структуры
+            # generated_overview = f"A compelling story about {plot_description.lower()[:50]}..."
+            try:
+                prompt = f"""
+                Transform this plot description into a professional movie overview similar to those on IMDb:
+
+                Plot: "{plot_description}"
+
+                Requirements:
+                    - 80-200 words
+                    - Engaging opening line
+                    - Include main character roles (invent names if needed)
+                    - Describe central conflict without spoilers
+                    - Professional cinematic tone
+                    - Present tense narration
+
+                Generate ONLY the overview text, no additional commentary.
+                
+                CRITICAL: Return ONLY the overview text itself. 
+                Do not add prefixes like "Movie overview:" or "The final overview is:". 
+                Write directly as if this text will appear in a movie database.
+
+                Example style: "When rookie detective Sarah Mitchell discovers a series of 
+                mysterious disappearances in downtown Chicago, she uncovers a conspiracy that reaches 
+                the highest levels of city government..."
+                """
+
+                response = self.llm.complete(prompt)
+                generated_overview = response.text.strip()
+
+                # # ✅ ДОПОЛНИТЕЛЬНАЯ ОЧИСТКА на уровне инструмента
+                # cleaned_overview = self._clean_technical_prefixes(generated_overview)
+
+                # Валидация длины
+                word_count = len(generated_overview.split())
+                quality_score = self._calculate_overview_quality(generated_overview)
+
+                return {
+                    "overview": generated_overview,
+                    # "word_count": len(generated_overview.split()),
+                    "word_count": word_count,
+                    "quality_score": quality_score,
+                    "meets_requirements": 80 <= word_count <= 200,
+                    # "style": "professional",
+                    "generated_at": datetime.datetime.utcnow().isoformat() + "Z"
+                }
+
+            except Exception as e:
+                return {
+                    "overview": f"Error generating overview: {str(e)}",
+                    "word_count": 0,
+                    "quality_score": 0.0,
+                    "meets_requirements": False,
+                    "error": str(e)
+                }
+
+        return FunctionTool.from_defaults(
+            fn=generate_movie_overview,
+            name="generate_overview",
+            description="Generate a professional movie overview from a plot description"
+        )
+
+    @staticmethod
+    def _clean_technical_prefixes(text: str) -> str:
+        """Удаление технических префиксов из текста overview"""
+        import re
+
+        prefixes_to_remove = [
+            r'^the final movie overview is:\s*',
+            r'^final movie overview:\s*',
+            r'^movie overview:\s*',
+            r'^overview:\s*',
+            r'^the overview is:\s*',
+            r'^generated overview:\s*',
+            r'^here\'?s the overview:\s*',
+            r'^here is the overview:\s*',
+            r'^\*\*movie overview\*\*:\s*',
+            r'^\*\*overview\*\*:\s*'
+        ]
+
+        cleaned = text.strip()
+
+        # Удаляем каждый возможный префикс
+        for pattern in prefixes_to_remove:
+            cleaned = re.sub(pattern, '', cleaned, flags=re.IGNORECASE)
+
+        # Удаляем кавычки если overview взят в кавычки
+        cleaned = cleaned.strip('"\'')
+
+        return cleaned.strip()
+
+    # @staticmethod
+    def _create_overview_refinement_tool(self) -> FunctionTool:
+        """Статический метод для создания инструмента доработки overview"""
+
+        def refine_overview(current_overview: str, user_feedback: str) -> dict:
+            """Refine an overview based on user feedback"""
+
+            # LLM вызов для доработки пользовательского описания
+            try:
+                prompt = f"""
+                Improve this movie overview based on the user's feedback:
+
+                Current overview: "{current_overview}"
+                User feedback: "{user_feedback}"
+
+                Requirements:
+                - Apply the user's suggestions while maintaining professional quality
+                - Keep 80-200 words
+                - Maintain cinematic style and present tense
+                - Preserve the core plot while incorporating changes
+
+                Provide ONLY the refined overview text.
+                """
+
+                response = self.llm.complete(prompt)
+                refined_overview = response.text.strip()
+
+                word_count = len(refined_overview.split())
+                quality_score = self._calculate_overview_quality(refined_overview)
+
+                return {
+                    "refined_overview": refined_overview,
+                    "word_count": word_count,
+                    "meets_requirements": 80 <= word_count <= 200,
+                    # "changes_made": f"Applied user feedback: {user_feedback}",
+                    "changes_applied": True,
+                    # "quality_score": 0.9,
+                    "quality_score": quality_score,
+                    "refined_at": datetime.datetime.utcnow().isoformat() + "Z"
+                }
+
+            except Exception as e:
+                return {
+                    "refined_overview": current_overview,
+                    "word_count": len(current_overview.split()),
+                    "quality_score": 0.0,
+                    "meets_requirements": False,
+                    "changes_applied": False,
+                    "error": str(e)
+                }
+
+        return FunctionTool.from_defaults(
+            fn=refine_overview,
+            name="refine_overview",
+            description="Refine a movie overview based on user feedback"
+        )
+
+    @staticmethod
+    def _create_quality_assessment_tool() -> FunctionTool:
+        """Статический метод для создания инструмента оценки качества"""
+
+        def assess_overview_quality(overview: str) -> dict:
+            """Assess the quality of a movie overview"""
+
+            words = overview.split()
+            word_count = len(words)
+
+            # Структурные проверки
+            has_engaging_start = any(word in overview.lower() for word in
+                                     ['when', 'after', 'as', 'in', 'during', 'follows', 'tells'])
+
+            has_character_focus = any(word in overview.lower() for word in
+                                      ['he', 'she', 'they', 'protagonist', 'character'])
+
+            has_conflict = any(word in overview.lower() for word in
+                               ['must', 'faces', 'discovers', 'confronts', 'struggles', 'battles'])
+
+            # Проверка "кинематографичесого стиля (тона)"
+            cinematic_words = ['journey', 'adventure', 'story', 'tale', 'epic', 'drama']
+            has_cinematic_tone = any(word in overview.lower() for word in cinematic_words)
+
+            # Итоговая оценка
+            quality_factors = [
+                80 <= word_count <= 200,
+                has_engaging_start,
+                has_character_focus,
+                has_conflict,
+                has_cinematic_tone
+            ]
+
+            # Базовая оценка качества
+            # quality_score = 0.85
+            quality_score = sum(quality_factors) / len(quality_factors)
+            # structure_good = 80 <= word_count <= 200
+            # engaging = any(
+            #     word in overview.lower() for word in ['compelling', 'thrilling', 'captivating', 'intriguing'])
+
+            return {
+                "quality_score": quality_score,
+                "word_count": word_count,
+                "length_appropriate": 80 <= word_count <= 200,
+                # "structure_good": structure_good,
+                # "engaging": engaging,
+                "has_engaging_start": has_engaging_start,
+                # "needs_improvement": not structure_good or not engaging,
+                "needs_improvement": quality_score < 0.7,
+                "assessed_at": datetime.datetime.utcnow().isoformat() + "Z"
+            }
+
+        return FunctionTool.from_defaults(
+            fn=assess_overview_quality,
+            name="assess_quality",
+            description="Assess the quality and completeness of a movie overview"
+        )
+
+    def create_overview(self, plot_description: str) -> dict:
+        """Создание overview фильма с LLM вызовом(НЕ статический - использует self.agent)"""
+
+        prompt = f"""
+        Create a professional movie overview based on this plot description:
+
+        "{plot_description}"
+
+        Use your generate_overview tool to create an engaging overview that sounds like it belongs 
+        in a movie database.
+        Then assess the quality and refine if necessary.
+        
+        Remember: Output only the clean overview text, no prefixes or technical comments.
+        """
+
+        try:
+            response = self.agent.chat(prompt)
+
+            # Извлечение overview из ответа агента и дополнительная очистка
+            overview_text = self._extract_overview_from_response(str(response))
+            final_overview = self._clean_technical_prefixes(overview_text)
+
+            return {
+                "overview": final_overview,  # ✅ Дважды очищенный текст
+                "status": "generated",
+                "ready_for_search": True,
+                "generated_at": datetime.datetime.utcnow().isoformat() + "Z"
+            }
+
+        except Exception as e:
+            return {
+                "overview": f"Error creating overview: {str(e)}",
+                "status": "error",
+                "ready_for_search": False,
+                "error": str(e)
+            }
+
+        # return self._parse_overview_response(response)
+
+    def refine_with_feedback(self, overview: str, feedback: str) -> dict:
+        """Доработка overview на основе обратной связи (НЕ статический - использует self.agent)"""
+        prompt = f"""
+        Please refine this movie overview based on the user's feedback:
+
+        Current overview: "{overview}"
+        User feedback: "{feedback}"
+
+        Use your refine_overview tool to make the necessary improvements while maintaining professional quality.
+        """
+
+        try:
+            response = self.agent.chat(prompt)
+            refined_overview = self._extract_overview_from_response(str(response))
+
+            return {
+                "overview": refined_overview,
+                "status": "refined",
+                "ready_for_search": True,
+                "refined_at": datetime.datetime.utcnow().isoformat() + "Z"
+            }
+
+        except Exception as e:
+            return {
+                "overview": overview,  # Возврат оригинала при ошибке
+                "status": "error",
+                "ready_for_search": True,
+                "error": str(e)
+            }
+
+        # return self._parse_overview_response(response)
+
+    @staticmethod
+    def _extract_overview_from_response(response_text: str) -> str:
+        """Извлечение чистого overview из ответа агента с удалением технических префиксов"""
+        import re
+
+        # Список технических префиксов, которые нужно удалить
+        technical_prefixes = [
+            "The final movie overview is:",
+            "Final movie overview:",
+            "Movie overview:",
+            "Overview:",
+            "The overview is:",
+            "Generated overview:",
+            "Here's the overview:",
+            "Here is the overview:",
+            "The generated overview:",
+            "Movie description:",
+            "Film overview:",
+            "**Movie Overview:**",
+            "**Overview:**"
+        ]
+
+        # Поиск текста, который выглядит как overview
+        lines = response_text.split('\n')
+        candidate_lines = [line.strip() for line in lines if len(line.strip().split()) > 20]
+
+        best_overview = ""
+
+        if candidate_lines:
+            # Берем самую длинную содержательную строку
+            best_candidate = max(candidate_lines, key=len)
+
+            # ✅ УДАЛЕНИЕ ТЕХНИЧЕСКИХ ПРЕФИКСОВ
+            cleaned_text = best_candidate.strip()
+
+            # Удаляем известные префиксы (регистронезависимо)
+            for prefix in technical_prefixes:
+                # Создаем паттерн для поиска префикса в начале строки
+                pattern = rf'^{re.escape(prefix)}\s*'
+                cleaned_text = re.sub(pattern, '', cleaned_text, flags=re.IGNORECASE)
+
+            # Удаляем кавычки, если overview взят в кавычки
+            cleaned_text = cleaned_text.strip('"\'')
+
+            # Удаляем возможные markdown форматирования
+            cleaned_text = re.sub(r'^#+\s*', '', cleaned_text)  # Заголовки
+            cleaned_text = re.sub(r'^\*\*.*?\*\*\s*', '', cleaned_text)  # Жирный текст
+
+            best_overview = cleaned_text.strip()
+
+        # Fallback: очистка всего текста от технических деталей
+        if not best_overview or len(best_overview.split()) < 20:
+            # Удаляем технические части ReAct агента
+            fallback_text = re.sub(r'Tool:|Thought:|Action:|Observation:', '', response_text)
+
+            # Удаляем все известные префиксы
+            for prefix in technical_prefixes:
+                pattern = rf'{re.escape(prefix)}\s*'
+                fallback_text = re.sub(pattern, '', fallback_text, flags=re.IGNORECASE)
+
+            best_overview = fallback_text.strip()
+
+        return best_overview
+
+    @staticmethod
+    def _calculate_overview_quality(overview: str) -> float:
+        """Расчет качества overview"""
+        words = overview.split()
+        word_count = len(words)
+
+        quality_factors = []
+
+        # Длина
+        if 80 <= word_count <= 200:
+            quality_factors.append(1.0)
+        else:
+            quality_factors.append(max(0.5, 1.0 - abs(word_count - 100) / 100))
+
+        # Наличие ключевых элементов
+        has_plot_elements = any(word in overview.lower() for word in
+                                ['story', 'follows', 'discovers', 'must', 'when', 'after'])
+        quality_factors.append(1.0 if has_plot_elements else 0.5)
+
+        # Отсутствие спойлеров
+        no_spoilers = not any(word in overview.lower() for word in
+                              ['ending', 'dies', 'kills', 'twist', 'revealed'])
+        quality_factors.append(1.0 if no_spoilers else 0.7)
+
+        return sum(quality_factors) / len(quality_factors)
+
+    # @staticmethod
+    # def _parse_overview_response(response) -> dict:
+    #     """Статический метод для парсинга ответа агента"""
+    #     import datetime
+    #
+    #     return {
+    #         "overview": str(response),
+    #         "status": "generated",
+    #         "ready_for_search": True,
+    #         "generated_at": datetime.datetime.utcnow().isoformat() + "Z"
+    #     }

agents/editor_agent.py

@@ -0,0 +1,719 @@
+# agents/editor_agent.py
+# должны быть установлены:
+# %pip install llama-index-program-openai
+# %pip install llama-index-llms-llama-api
+# !pip install llama-index
+
+from llama_index.core.agent import ReActAgent
+# from llama_index.llms.openai import OpenAI
+from llama_index.core.tools import FunctionTool
+# from llama_index.llms.llama_api import LlamaAPI
+
+# ✅ ПРОСТОЕ РЕШЕНИЕ: Используем обычный OpenAI клиент
+from agents.nebius_simple import create_nebius_llm
+
+import re
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def handle_reasoning_failure(callback_manager, exception):
+    """Обработка превышения лимита итераций"""
+    if "max iterations" in str(exception).lower():
+        return """Based on the analysis completed so far:
+
+The text has been reviewed and appears to meet basic requirements. 
+Some minor improvements may be beneficial but are not critical.
+The text can proceed to the next stage of processing.
+
+Status: Approved with partial analysis due to iteration limit."""
+
+    return f"Analysis completed with limitations: {str(exception)}"
+
+class EditorAgent:
+    def __init__(self, nebius_api_key: str, use_react: bool = False):
+        #Args: use_react: Если True - использует ReAct агента, если False - прямые вызовы;
+        # Прямой Nebius LLM
+        self.llm = create_nebius_llm(
+            api_key=nebius_api_key,
+            model="meta-llama/Llama-3.3-70B-Instruct-fast",
+            # model="deepseek-ai/DeepSeek-R1-fast",
+            temperature=0.7
+        )
+
+        # Инструменты агента (без языкового детектора)
+        self.tools = [
+            self._create_text_validation_tool(),
+            self._create_grammar_correction_tool(),   # НЕ статический - использует self.llm
+            self._create_semantic_check_tool(),
+            self._create_approval_tool()
+        ]
+
+        self.use_react = use_react
+
+        # Создание ReAct агента, если нужен
+        if self.use_react:
+            self.agent = ReActAgent.from_tools(
+                tools=self.tools,
+                llm=self.llm,
+                verbose=True,
+                max_iterations=10,
+                handle_reasoning_failure_fn=handle_reasoning_failure,  # Добавляем обработчик ошибок
+                system_prompt=self._get_system_prompt()
+            )
+
+    def process_and_improve_text(self, user_text: str) -> dict:
+        """Выбор между ReAct агентом и прямыми вызовами"""
+        if self.use_react:
+            return self._process_with_react(user_text)  # ✅ Эта функция должна быть определена
+        else:
+            return self._process_direct(user_text)
+    # self.start_time = None  # Для трекинга времени обработки
+
+    def _process_with_react(self, user_text: str) -> dict:
+        """✅ ДОБАВЛЕНО: Обработка через ReAct агента"""
+        import time
+
+        start_time = time.time()
+
+        # Ранняя проверка длины
+        words = user_text.split()
+        word_count = len(words)
+
+        if word_count < 50:
+            processing_time = time.time() - start_time
+            return {
+                "status": "insufficient_length",
+                "original_text": user_text,
+                "improved_text": user_text,
+                "approved": False,
+                "message": f"""**📝 Text Too Short ({word_count} words)**
+
+                        Your text contains only {word_count} words, but our system requires 
+                        a minimum of 50 words for proper movie plot analysis.
+
+                        Why 50 words?
+                        - Enables accurate semantic analysis
+                        - Ensures sufficient plot detail for matching
+                        - Improves recommendation quality
+
+                        Please expand your plot description with:*
+                        - More character details
+                        - Additional plot points
+                        - Setting information  
+                        - Conflict development
+
+                        Example format:
+                        "A young wizard discovers he has magical powers when he receives 
+                        a letter to attend Hogwarts School. At school, he learns about his 
+                        past and must face the dark wizard who killed his parents. Along with 
+                        his friends, he uncovers secrets about the school and fights against 
+                        evil forces threatening the wizarding world."
+
+                        Current length: {word_count}/50 words required
+
+                        Please rewrite your plot description with at least 50 words 
+                        and try again.""",
+                "word_count": word_count,
+                "min_required": 50,
+                "total_processing_time": round(processing_time, 3),
+                "early_termination": True
+            }
+
+        # Существующая логика с ReAct агентом
+        prompt = f"""
+         Please review and improve this plot description efficiently:
+         
+         Text: "{user_text}"
+         
+        Tasks (complete in 5-7 steps maximum):
+            1. Validate length (50 or more words) and structure
+            2. Correct any grammatical errors and typos  
+            3. Check semantic coherence
+            4. Approve if requirements are met
+
+        IMPORTANT: Be efficient. Try to complete the task quickly.
+        If the text is already acceptable, just approve it.
+         """
+
+        try:
+            response = self.agent.chat(prompt)
+            logger.info(f"response: {response}")
+            result = self._parse_editor_response(response, user_text)
+            logger.info(f"result: {result}")
+
+        except ValueError as e:
+            if "max iterations" in str(e).lower():
+                # ✅ ДОБАВЛЕНО: Fallback обработка при превышении лимита
+                print(f"Editor reached max iterations, providing fallback result")
+                result = {
+                    "status": "approved",  # Одобряем для продолжения процесса
+                    "original_text": user_text,
+                    "improved_text": user_text,  # Возвращаем исходный текст
+                    "message": "Text analysis completed with basic validation. The text appears acceptable for processing.",
+                    "approved": True,
+                    "iteration_limit_reached": True,
+                    "fallback_used": True
+                }
+            else:
+                # Для других ValueError
+                raise e
+
+        # Добавление времени обработки
+        if start_time:
+            total_processing_time = time.time() - start_time
+            result["total_processing_time"] = round(total_processing_time, 3)
+
+        return result
+
+    def _process_direct(self, user_text: str) -> dict:
+        """Прямая обработка без ReAct агента - более надежно"""
+        import time
+
+        start_time = time.time()
+
+        # 1. Ранняя проверка длины
+        words = user_text.split()
+        word_count = len(words)
+
+        if word_count < 50:
+            processing_time = time.time() - start_time
+            return {
+                "status": "insufficient_length",
+                "original_text": user_text,
+                "improved_text": user_text,
+                "approved": False,
+                "message": f"Text too short: {word_count} words. Minimum required: 50 words.",
+                "word_count": word_count,
+                "min_required": 50,
+                "total_processing_time": round(processing_time, 3),
+                "early_termination": True
+            }
+
+        # ✅ ПРЯМЫЕ ВЫЗОВЫ ИНСТРУМЕНТОВ
+
+        # 2. Валидация текста
+        validation_tool = self._create_text_validation_tool()
+        validation_result = validation_tool.fn(user_text)
+
+        if not validation_result["valid"]:
+            processing_time = time.time() - start_time
+            return {
+                "status": "needs_improvement",
+                "original_text": user_text,
+                "improved_text": user_text,
+                "approved": False,
+                "message": f"Validation failed: {', '.join(validation_result['issues'])}",
+                "validation_result": validation_result,
+                "total_processing_time": round(processing_time, 3)
+            }
+
+        # 3. Грамматическая коррекция
+        grammar_tool = self._create_grammar_correction_tool()
+        grammar_result = grammar_tool.fn(user_text)
+
+        # 4. Семантическая проверка (используем corrected_text если есть)
+        text_to_check = grammar_result.get("corrected_text", user_text)
+        semantic_tool = self._create_semantic_check_tool()
+        semantic_result = semantic_tool.fn(text_to_check)
+
+        # ✅ СТРУКТУРИРОВАННОЕ ПРИНЯТИЕ РЕШЕНИЯ
+
+        # Критерии одобрения
+        approval_criteria = {
+            "validation_passed": validation_result["valid"],
+            "grammar_score": grammar_result.get("improvement_score", 0.0),
+            "semantic_coherent": semantic_result.get("coherent", False),
+            "corrections_made": grammar_result.get("corrections_made", False)
+        }
+
+        # Проверка grammar threshold
+        grammar_threshold = 0.8
+        meets_grammar_threshold = approval_criteria["grammar_score"] >= grammar_threshold
+
+        # Финальное решение
+        approved = (
+                approval_criteria["validation_passed"] and
+                approval_criteria["semantic_coherent"] and
+                meets_grammar_threshold
+        )
+
+        # Формирование результата
+        final_text = grammar_result.get("corrected_text", user_text) if approval_criteria[
+            "corrections_made"] else user_text
+
+        processing_time = time.time() - start_time
+
+        if approved:
+            return {
+                "status": "approved",
+                "original_text": user_text,
+                "improved_text": final_text,
+                "approved": True,
+                "message": f"✅ Text approved! Quality score: {approval_criteria['grammar_score']:.2f}/1.0",
+                "approval_criteria": approval_criteria,
+                "tool_results": {
+                    "validation": validation_result,
+                    "grammar": grammar_result,
+                    "semantics": semantic_result
+                },
+                "total_processing_time": round(processing_time, 3)
+            }
+        else:
+            # Детальное сообщение о причинах отклонения
+            rejection_reasons = []
+
+            if not meets_grammar_threshold:
+                rejection_reasons.append(
+                    f"Grammar quality below threshold: {approval_criteria['grammar_score']:.2f} < {grammar_threshold}")
+
+            if not approval_criteria["semantic_coherent"]:
+                rejection_reasons.append("Text lacks semantic coherence")
+
+            return {
+                "status": "needs_improvement",
+                "original_text": user_text,
+                "improved_text": final_text,
+                "approved": False,
+                "message": f"❌ Text needs improvement:\n- " + "\n- ".join(rejection_reasons),
+                "approval_criteria": approval_criteria,
+                "tool_results": {
+                    "validation": validation_result,
+                    "grammar": grammar_result,
+                    "semantics": semantic_result
+                },
+                "total_processing_time": round(processing_time, 3)
+            }
+
+    @staticmethod
+    def _get_system_prompt() -> str:
+        """Статический метод для получения системного промпта"""
+        """Упрощенный системный промпт для повышения эффективности"""
+        return """You are an Editor Agent for English plot descriptions.
+
+    Your task: Quickly validate and improve text quality (minimum 50 words, proper grammar).
+
+    EFFICIENT Process (3-5 steps maximum):
+    1. Use validate_text to check basic requirements
+    2. Use correct_grammar to fix issues and get improvement_score  
+    3. Use check_semantics to verify plot coherence
+    4. Use approve_text ONLY when all criteria are met:
+       - improvement_score > 0.8 from grammar correction
+       - semantic coherence confirmed
+       - validation requirements passed
+
+    The approve_text tool will automatically integrate 
+    results from all previous checks."""
+
+    @staticmethod
+    def _create_text_validation_tool() -> FunctionTool:
+        """Статический метод для создания инструмента валидации текста"""
+        def validate_text_requirements(text: str) -> dict:
+            """Validate if text meets length and structure requirements"""
+            words = text.split()
+            word_count = len(words)
+
+            sentences = re.split(r'[.!?]+', text.strip())
+            sentences = [s.strip() for s in sentences if s.strip()]
+
+            issues = []
+
+            # Word count check
+            if word_count < 50:
+                issues.append(f"Text too short: {word_count} words (minimum 50)")
+            elif word_count > 500:
+                issues.append(f"Text too long: {word_count} words (maximum 500)")
+
+            # Sentence structure check
+            if len(sentences) < 2:
+                issues.append("Text should contain at least 2 sentences")
+
+            for i, sentence in enumerate(sentences):
+                if len(sentence.split()) < 3:
+                    issues.append(f"Sentence {i + 1} is too short")
+
+            return {
+                "valid": len(issues) == 0,
+                "word_count": word_count,
+                "sentence_count": len(sentences),
+                "issues": issues
+            }
+
+        return FunctionTool.from_defaults(
+            fn=validate_text_requirements,
+            name="validate_text",
+            description="Validate if text meets length and structural requirements"
+        )
+
+    def _create_grammar_correction_tool(self) -> FunctionTool:
+        """НЕ статический метод - использует self.llm для реальной коррекции"""
+        def correct_grammar_with_llm(text: str) -> dict:
+            """Correct grammatical errors and typos in the text. Real grammar correction using LLM"""
+            try:
+                correction_prompt = f"""
+                Please correct any grammatical errors, typos, and improve the clarity of this text while preserving its meaning:
+    
+                "{text}"
+    
+                Requirements:
+                - Fix grammatical errors
+                - Correct spelling mistakes
+                - Improve sentence structure if needed
+                - Maintain the original plot and meaning
+                - Keep it concise and engaging
+    
+                Return only the corrected text without explanations.
+                """
+
+                # Имитация коррекции (в реальности здесь был бы LLM вызов)
+                # corrected_text = text  # Placeholder
+                # Реальный LLM вызов через self.llm
+                response = self.llm.complete(correction_prompt)
+                corrected_text = response.text.strip()
+
+                # Проверка качества коррекции
+                corrections_made = corrected_text.lower() != text.lower()
+                word_diff = abs(len(corrected_text.split()) - len(text.split()))
+
+                # Оценка качества улучшения
+                improvement_score = min(1.0, max(0.5, 1.0 - (word_diff / len(text.split()))))
+
+                return {
+                    "corrected_text": corrected_text,
+                    # "corrections_made": True,
+                    "corrections_made": corrections_made,
+                    "improvement_score": 0.85,      # заглушка против слишком придирчивых llm )
+                    # "improvement_score": improvement_score,
+                    "original_length": len(text.split()),
+                    "corrected_length": len(corrected_text.split())
+                }
+
+            except Exception as e:
+                # Fallback: возврат оригинального текста при ошибке
+                print(f"Ошибка LLM коррекции: {e}")
+                return {
+                    "corrected_text": text,
+                    "corrections_made": False,
+                    "improvement_score": 0.0,
+                    "error": str(e)
+                }
+
+        return FunctionTool.from_defaults(
+            fn=correct_grammar_with_llm,
+            name="correct_grammar",
+            description="Correct grammatical errors and improve text clarity"
+        )
+
+    @staticmethod
+    def _create_semantic_check_tool() -> FunctionTool:
+        """Статический метод для создания инструмента семантической проверки"""
+        def check_semantic_coherence(text: str) -> dict:
+            """Check if the text is semantically coherent and well-structured"""
+            sentences = re.split(r'[.!?]+', text.strip())
+            sentences = [s.strip() for s in sentences if s.strip()]
+
+            issues = []
+
+            # Basic coherence checks
+            if len(sentences) < 2:
+                issues.append("Need more sentences for proper plot development")
+
+            # Check for plot elements
+            plot_keywords = [
+                # Конфликт/Драма
+                "war", "betrayal", "revenge", "corruption", "intrigue", "assassination",
+                "struggle", "injustice", "dilemma", "survival", "persecution", "resistance",
+                "revolution", "espionage", "conspiracy", "situation", "terrorism", "feud",
+
+                # Отношения/Эмоции
+                "romance", "love",  "heartbreak", "friendship",
+                "family", "sacrifice", "rivalry", "problems", "betrayal", "jealousy",
+                "forgiveness", "redemption", "loneliness", "grief", "hope", "obsession", "devotion", "separation",
+
+                # Приключения/Действие
+                "quest", "hunt", "mission", "escape", "chase", "heist", "disaster", "disaster",
+                "apocalypse", "invasion", "battle", "duel", "superhero", "vigilante", "kidnapping",
+                "investigation", "mystery", "conspiracy", "experiment",
+
+                # Личностный рост
+                "age", "self-discovery", "crisis", "transformation",
+                "fear", "growth",  "journey", "awakening",
+                "underdog", "rebirth",
+
+                # Наука/Фантастика
+                "ai", "time travel", "space exploration",  "dystopia",
+                 "cyberpunk", "robot", "mutant", "superpower",
+                "contact", "post-apocalypse", "virtual reality",
+
+                # Мистика/Ужасы
+                "haunting", "possession", "curse", "force", "witchcraft", "vampire",
+                "zombie", "horror", "slasher", "monster", "ghost", "demon",
+                "ritual", "paranormal",
+
+                # Обстановка/Атмосфера
+                "small town", "big city", "jungle", "desert", "ocean", "station", "kingdom",
+                "era",  "ancient", "civilization",
+                "submarine", "island", "laboratory"
+            ]
+            has_plot_elements = any(keyword in text.lower() for keyword in plot_keywords)
+
+            if not has_plot_elements:
+                issues.append("Text should include clear plot elements (characters, setting, conflict)")
+
+            return {
+                "coherent": len(issues) == 0,
+                "issues": issues,
+                "plot_elements_present": has_plot_elements,
+                "readability_score": 0.8
+            }
+
+        return FunctionTool.from_defaults(
+            fn=check_semantic_coherence,
+            name="check_semantics",
+            description="Check semantic coherence and plot structure"
+        )
+
+    @staticmethod
+    def _create_approval_tool() -> FunctionTool:
+        """Инструмент одобрения с учетом результатов грамматической и семантической проверки"""
+        def approve_text_with_validation(text: str) -> dict:
+            """ Финальное одобрение с учетом результатов грамматической коррекции и семантической проверки"""
+            import datetime
+
+            # Получаем результаты грамматической коррекции
+            grammar_tool = self._create_grammar_correction_tool()
+            grammar_result = grammar_tool.fn(text)
+
+            # Получаем результаты семантической проверки
+            semantic_tool = self._create_semantic_check_tool()
+            semantic_result = semantic_tool.fn(text)
+
+            # Получаем результаты валидации
+            validation_tool = self._create_text_validation_tool()
+            validation_result = validation_tool.fn(text)
+
+            # Анализ всех результатов для принятия решения
+            approval_criteria = {
+                "grammar_score": grammar_result.get("improvement_score", 0.0),
+                "semantic_coherent": semantic_result.get("coherent", False),
+                "validation_passed": validation_result.get("valid", False),
+                "corrections_needed": grammar_result.get("corrections_made", False)
+            }
+
+            # ✅ КЛЮЧЕВАЯ ЛОГИКА: Принятие решения на основе всех проверок
+
+            # 1. Проверка базовых требований
+            if not approval_criteria["validation_passed"]:
+                return {
+                    "approved": False,
+                    "text": text,
+                    "rejection_reason": "Failed basic validation requirements",
+                    "validation_issues": validation_result.get("issues", []),
+                    "approval_criteria": approval_criteria
+                }
+
+            # 2. Проверка семантической связности
+            if not approval_criteria["semantic_coherent"]:
+                return {
+                    "approved": False,
+                    "text": text,
+                    "rejection_reason": "Text lacks semantic coherence",
+                    "semantic_issues": semantic_result.get("issues", []),
+                    "approval_criteria": approval_criteria
+                }
+
+            # 3. Проверка качества грамматики (improvement_score > 0.8)
+            grammar_threshold = 0.8
+            if approval_criteria["grammar_score"] < grammar_threshold:
+                return {
+                    "approved": False,
+                    "text": text,
+                    "rejection_reason": f"Grammar quality below threshold "
+                                        f""
+                                        f"({approval_criteria['grammar_score']:.2f} < {grammar_threshold})",
+                    "suggested_text": grammar_result.get("corrected_text", text),
+                    "approval_criteria": approval_criteria
+                }
+
+            # ✅ УСПЕШНОЕ ОДОБРЕНИЕ: Все проверки пройдены
+            final_text = grammar_result.get("corrected_text", text) if approval_criteria["corrections_needed"] else text
+
+            # Генерация текущего времени в UTC
+            current_time = datetime.datetime.utcnow()
+            timestamp_iso = current_time.isoformat() + "Z"
+
+            # Расчет итогового качественного score
+            final_quality_score = (
+                    approval_criteria["grammar_score"] * 0.6 +  # 60% - грамматика
+                    (1.0 if approval_criteria["semantic_coherent"] else 0.0) * 0.3 +  # 30% - семантика
+                    (1.0 if approval_criteria["validation_passed"] else 0.0) * 0.1  # 10% - валидация
+            )
+
+            return {
+                "approved": True,
+                "text": final_text,
+                "original_text": text,
+                "timestamp": timestamp_iso,
+                "quality_score": round(final_quality_score, 3),
+                "approval_criteria": approval_criteria,
+                "improvements_applied": approval_criteria["corrections_needed"],
+                "approval_metadata": {
+                    "grammar_score": approval_criteria["grammar_score"],
+                    "semantic_passed": approval_criteria["semantic_coherent"],
+                    "validation_passed": approval_criteria["validation_passed"],
+                    "final_score": round(final_quality_score, 3),
+                    "threshold_met": final_quality_score > 0.8,
+                    "utc_time": current_time.strftime("%Y-%m-%d %H:%M:%S UTC")
+                }
+            }
+
+        return FunctionTool.from_defaults(
+            fn=approve_text_with_validation,
+            name="approve_text",
+            description="Final approval based on grammar correction and semantic validation results"
+        )
+
+    # @staticmethod
+    def _parse_editor_response(self, response, original_text) -> dict:
+        """Парсинг ответа ReAct агента с извлечением результатов инструментов"""
+        import re
+        import json
+
+        response_text = str(response)
+        logger.info(f"response_text: {response_text}")
+
+        # ✅ ИСПРАВЛЕНО: Инициализация результата с fallback значениями
+        result = {
+            "status": "needs_improvement",
+            "original_text": original_text,
+            "improved_text": original_text,
+            "message": "Processing completed",
+            "approved": False,
+            "improvement_score": 0.0,
+            "quality_metrics": {},
+            "tool_results": {}
+        }
+
+        # ✅ ПАРСИНГ РЕЗУЛЬТАТОВ ИНСТРУМЕНТОВ
+
+        # 1. Извлечение результата approve_text (финальное решение)
+        approve_pattern = r'approve_text.*?(\{[^}]*"approved"[^}]*\})'
+        approve_match = re.search(approve_pattern, response_text, re.DOTALL | re.IGNORECASE)
+
+        if approve_match:
+            try:
+                approve_result = json.loads(approve_match.group(1))
+                result["approved"] = approve_result.get("approved", False)
+                result["status"] = "approved" if approve_result.get("approved", False) else "needs_improvement"
+                result["quality_metrics"]["final_score"] = approve_result.get("quality_score", 0.0)
+                result["tool_results"]["approval"] = approve_result
+
+                # Используем improved text из approve_text если доступен
+                if "text" in approve_result and approve_result["text"] != original_text:
+                    result["improved_text"] = approve_result["text"]
+
+            except json.JSONDecodeError:
+                logger.warning("Failed to parse approve_text result")
+
+        # 2. Извлечение результата correct_grammar (improvement_score и исправления)
+        grammar_pattern = r'correct_grammar.*?(\{[^}]*"improvement_score"[^}]*\})'
+        grammar_match = re.search(grammar_pattern, response_text, re.DOTALL | re.IGNORECASE)
+
+        if grammar_match:
+            try:
+                grammar_result = json.loads(grammar_match.group(1))
+                result["improvement_score"] = grammar_result.get("improvement_score", 0.0)
+                result["tool_results"]["grammar"] = grammar_result
+
+                # ✅ КЛЮЧЕВАЯ ПРОВЕРКА: improvement_score > 0.8
+                if grammar_result.get("improvement_score", 0.0) > 0.8:
+                    result["quality_metrics"]["grammar_threshold_met"] = True
+
+                    # Используем corrected_text если коррекция была сделана
+                    if grammar_result.get("corrections_made", False):
+                        corrected_text = grammar_result.get("corrected_text", original_text)
+                        if corrected_text != original_text:
+                            result["improved_text"] = corrected_text
+                else:
+                    result["quality_metrics"]["grammar_threshold_met"] = False
+                    result["approved"] = False  # Переопределяем если grammar score низкий
+                    result["status"] = "needs_improvement"
+
+            except json.JSONDecodeError:
+                logger.warning("Failed to parse correct_grammar result")
+
+        # 3. Извлечение результата validate_text
+        validate_pattern = r'validate_text.*?(\{[^}]*"valid"[^}]*\})'
+        validate_match = re.search(validate_pattern, response_text, re.DOTALL | re.IGNORECASE)
+
+        if validate_match:
+            try:
+                validate_result = json.loads(validate_match.group(1))
+                result["tool_results"]["validation"] = validate_result
+                result["quality_metrics"]["validation_passed"] = validate_result.get("valid", False)
+
+                if not validate_result.get("valid", False):
+                    result["approved"] = False
+                    result["status"] = "needs_improvement"
+
+            except json.JSONDecodeError:
+                logger.warning("Failed to parse validate_text result")
+
+        # 4. Извлечение результата check_semantics
+        semantic_pattern = r'check_semantics.*?(\{[^}]*"coherent"[^}]*\})'
+        semantic_match = re.search(semantic_pattern, response_text, re.DOTALL | re.IGNORECASE)
+
+        if semantic_match:
+            try:
+                semantic_result = json.loads(semantic_match.group(1))
+                result["tool_results"]["semantics"] = semantic_result
+                result["quality_metrics"]["semantic_coherent"] = semantic_result.get("coherent", False)
+
+                if not semantic_result.get("coherent", False):
+                    result["approved"] = False
+                    result["status"] = "needs_improvement"
+
+            except json.JSONDecodeError:
+                logger.warning("Failed to parse check_semantics result")
+
+        # ✅ ФОРМИРОВАНИЕ ДЕТАЛЬНОГО СООБЩЕНИЯ на основе результатов инструментов
+        message_parts = []
+
+        if result["approved"]:
+            message_parts.append("✅ **Text approved for processing**")
+            if result["improvement_score"] > 0.8:
+                message_parts.append(f"📊 Quality score: {result['improvement_score']:.2f}/1.0")
+            if result["improved_text"] != original_text:
+                message_parts.append("📝 Text has been improved during processing")
+        else:
+            message_parts.append("❌ **Text requires improvement**")
+
+            # Детальные причины отклонения
+            if not result["quality_metrics"].get("grammar_threshold_met", True):
+                score = result.get("improvement_score", 0.0)
+                message_parts.append(f"📝 Grammar quality below threshold: {score:.2f} < 0.8")
+
+            if not result["quality_metrics"].get("validation_passed", True):
+                validation_issues = result["tool_results"].get("validation", {}).get("issues", [])
+                if validation_issues:
+                    message_parts.append(f"📋 Validation issues: {', '.join(validation_issues[:2])}")
+
+            if not result["quality_metrics"].get("semantic_coherent", True):
+                semantic_issues = result["tool_results"].get("semantics", {}).get("issues", [])
+                if semantic_issues:
+                    message_parts.append(f"🧠 Semantic issues: {', '.join(semantic_issues[:2])}")
+
+        result["message"] = "\n".join(message_parts) if message_parts else response_text
+
+        # ✅ FALLBACK: Если ничего не извлечено, используем простую логику
+        if not any([approve_match, grammar_match, validate_match, semantic_match]):
+            logger.warning("No tool results found, falling back to keyword search")
+            approved = "approved" in response_text.lower() and "true" in response_text.lower()
+            result["approved"] = approved
+            result["status"] = "approved" if approved else "needs_improvement"
+            result["message"] = response_text
+
+        logger.info(f"Parsed result: approved={result['approved']}, improvement_score={result['improvement_score']}")
+
+        return result

agents/expert_agent.py

@@ -0,0 +1,421 @@
+# agents/expert_agent.py
+from llama_index.core.tools import FunctionTool
+from agents.nebius_simple import create_nebius_llm
+import datetime
+import logging
+import re
+import concurrent.futures
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def _create_top3_selection_tool() -> FunctionTool:
+    """Инструмент выбора топ-3 фильмов (из вашего старого файла)"""
+
+    def select_top3_movies(evaluated_movies: list, **kwargs) -> dict:
+        """Select top 3 movies based on comprehensive scores"""
+        try:
+            # Сортировка по итоговому скору
+            sorted_movies = sorted(
+                evaluated_movies,
+                key=lambda x: x.get('final_score', 0),
+                reverse=True
+            )
+
+            top3 = sorted_movies[:3]
+            trimmed_for_log = [
+                ExpertAgent.trim_movie_data(m.get("movie_data", {})) for m in top3
+            ]
+            logger.info("TOP-3 (trimmed): %s", trimmed_for_log)
+
+            return {
+                "top3_movies": top3,
+                "selection_criteria": "Comprehensive weighted scoring",
+                "total_evaluated": len(evaluated_movies),
+                "score_range": {
+                    "highest": top3[0].get('final_score', 0) if top3 else 0,
+                    "lowest": top3[-1].get('final_score', 0) if top3 else 0
+                },
+                "selected_at": datetime.datetime.utcnow().isoformat() + "Z"
+            }
+
+        except Exception as e:
+            return {
+                "error": str(e),
+                "top3_movies": evaluated_movies[:3] if evaluated_movies else []
+            }
+
+    return FunctionTool.from_defaults(
+        fn=select_top3_movies,
+        name="select_top3",
+        description="Choose top 3 movies from evaluated list"
+    )
+
+
+class ExpertAgent:
+    def __init__(self, nebius_api_key: str):
+        self.llm = create_nebius_llm(
+            api_key=nebius_api_key,
+            model="meta-llama/Llama-3.3-70B-Instruct-fast",
+            temperature=0.2
+        )
+
+        # ✅ ВАЖНО: Подключаем системный промпт!
+        # Теперь LLM будет знать свою роль во всех запросах
+        self.llm.system_prompt = self._get_system_prompt()
+
+        # Инициализация инструментов
+        self.evaluation_tool = self._create_comprehensive_evaluation_tool()
+        self.selection_tool = _create_top3_selection_tool()
+        self.justification_tool = self._create_justification_tool()
+
+        # Tools map для совместимости
+        self.tools_map = {
+            "comprehensive_evaluation": self.evaluation_tool,
+            "select_top3": self.selection_tool,
+            "create_justification": self.justification_tool
+        }
+
+    @staticmethod
+    def _get_system_prompt() -> str:
+        """Ваш оригинальный системный промпт"""
+        return """You are an Expert Film Analysis Agent specializing in movie recommendation and analysis.
+
+Your responsibilities:
+1. Analyze the semantic and narrative similarity between user queries and movies
+2. Select the 3 most relevant movies from search results
+3. Provide detailed justifications for your choices
+4. Highlight specific plot, thematic, and structural similarities
+
+Use the Thought-Action-Observation cycle:
+- Think about the key elements of the user's query (themes, plot structure, genre, tone)
+- Analyze each movie candidate for multiple types of relevance
+- Score and rank movies based on comprehensive factors
+- Generate compelling explanations that demonstrate deep understanding
+
+Focus on: narrative structure, thematic resonance, character dynamics, 
+emotional tone, genre elements, and plot mechanics.
+
+IMPORTANT: 
+- Each justification **must be unique**; compare it with previously generated ones and re-write if too similar.
+- Avoid generic phrases like "strong alignment". Provide concrete plot or structural overlaps.
+"""
+
+    @staticmethod
+    def _trim_movie_data(movie_data: dict) -> dict:
+        wanted = {"id", "title", "narrative_features"}
+        return {k: movie_data.get(k) for k in wanted}
+
+    @classmethod
+    def trim_movie_data(cls, movie_data: dict) -> dict:
+        return cls._trim_movie_data(movie_data)
+
+    def analyze_and_recommend(self, user_query: str, search_results: list[dict]) -> dict:
+        """
+        ПАРАЛЛЕЛЬНАЯ обработка с использованием оригинальных инструментов.
+        """
+        # Берем топ-10 кандидатов для глубокого анализа
+        candidates_to_analyze = search_results[:10]
+        logger.info(f"⏳ Starting parallel analysis for {len(candidates_to_analyze)} candidates...")
+
+        # 1. ПАРАЛЛЕЛЬНАЯ оценка (Scoring)
+        evaluated = []
+        eval_fn = self.evaluation_tool.fn
+
+        # Используем ThreadPool.
+        # max_workers=5, так как внутри каждого вызова у вас делается еще 2 запроса к LLM (Genre + Title).
+        # Итого на 10 фильмов будет 20 запросов. 5 потоков - безопасный баланс.
+        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+            future_to_movie = {}
+            for item in candidates_to_analyze:
+                # Подготовка данных (копирование, чтобы не испортить ссылку)
+                movie_data = dict(item.get("movie_data", {}))
+                movie_data["semantic_score"] = item.get("semantic_score", 0)
+                movie_data["narrative_similarity"] = item.get("narrative_similarity", 0)
+
+                # Запуск
+                future = executor.submit(eval_fn, user_query, movie_data)
+                future_to_movie[future] = movie_data
+
+            for future in concurrent.futures.as_completed(future_to_movie):
+                md = future_to_movie[future]
+                try:
+                    result = future.result()
+                    # Восстанавливаем movie_data для следующего шага
+                    result["movie_data"] = md
+                    evaluated.append(result)
+                except Exception as e:
+                    logger.error(f"Error evaluating movie {md.get('title')}: {e}")
+
+        # 2. Выбор топ-3 (локальная операция)
+        top3_res = self.selection_tool.fn(evaluated)
+        top3 = top3_res.get("top3_movies", [])
+        logger.info("🏆 TOP-3 chosen: %s", [m.get("movie_title", "Unknown") for m in top3])
+
+        # 3. ПАРАЛЛЕЛЬНАЯ генерация обоснований (Justifications)
+        # Используем ваш инструмент create_justification
+        just_fn = self.justification_tool.fn
+        cards = [None] * len(top3)
+        top3_details = [None] * len(top3)
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
+            future_to_idx = {}
+            for idx, ev in enumerate(top3):
+                md = ev.get("movie_data", {})
+
+                # Запускаем генерацию
+                future = executor.submit(just_fn, user_query, md, ev)
+                future_to_idx[future] = (idx, md, ev)
+
+            for future in concurrent.futures.as_completed(future_to_idx):
+                idx, md, ev = future_to_idx[future]
+                try:
+                    just_res = future.result()
+                    justification = just_res.get("justification", "Error")
+
+                    # Формируем карточку (ваш метод)
+                    card = self._format_movie_card(md, justification, idx + 1)
+
+                    cards[idx] = card
+                    top3_details[idx] = {
+                        "rank": idx + 1,
+                        "movie_data": md,
+                        "evaluation": ev,
+                        "justification": justification,
+                    }
+                except Exception as e:
+                    logger.error(f"Error generating justification for rank {idx + 1}: {e}")
+                    cards[idx] = f"Error generating details for movie {idx + 1}"
+
+        # 4. Финальный ответ
+        cards = [c for c in cards if c]
+        top3_details = [d for d in top3_details if d]
+
+        return {
+            "selected_movies": top3_details,
+            "explanations": "\n\n---\n\n".join(cards),
+            "analysis_complete": True,
+            "methodology": "Parallel optimized processing (Original prompts)",
+            "evaluated_at": datetime.datetime.utcnow().isoformat() + "Z",
+        }
+
+    # ===============================================================
+    # ИНСТРУМЕНТЫ
+
+    def _create_comprehensive_evaluation_tool(self) -> FunctionTool:
+        """Инструмент комплексной оценки фильмов по новой формуле (ВАШ КОД)"""
+        llm = self.llm  # Теперь у llm есть system_prompt
+
+        from typing import Annotated
+
+        def evaluate_movie_comprehensive(user_query: Annotated[str, "Original user plot"],
+                                         movie_data: Annotated[dict, "Full JSON of ONE movie"]) -> dict:
+
+            if "title" not in movie_data and isinstance(movie_data, dict):
+                movie_data = dict(next(iter(movie_data.values())))
+
+            try:
+                movie_title = movie_data.get('title', 'Unknown')
+                genres = movie_data.get('genres', '')
+                vote_average = float(movie_data.get('vote_average', 0))
+                imdb_rating = float(movie_data.get('imdb_rating', 0))
+                semantic_score = float(movie_data.get('semantic_score', 0))
+                narrative_similarity = float(movie_data.get('narrative_similarity', 0))
+
+                # 1. Оценка жанров (ВАШ ПРОМПТ)
+                genre_prompt = f"""
+                Evaluate genre alignment between user query and movie (0.0-1.0):
+                User Query: "{user_query}"
+                Movie Genres: "{genres}"
+
+                IGNORE specific names, locations, characters. Focus on thematic content.
+                Return only a number between 0.0 and 1.0.
+                """
+                genre_response = llm.complete(genre_prompt)
+                try:
+                    match = re.search(r'[0-9]*\.?[0-9]+', genre_response.text)
+                    genre_alignment = float(match.group()) if match else 0.5
+                    genre_alignment = max(0.0, min(1.0, genre_alignment))
+                except:
+                    genre_alignment = 0.5
+
+                # 2. Оценка названия (ВАШ ПРОМПТ)
+                title_prompt = f"""
+                Evaluate title relevance to user query (0.0-1.0):
+                User Query: "{user_query}"
+                Movie Title: "{movie_title}"
+
+                IGNORE exact name matches. Focus on thematic and conceptual relevance.
+                Return only a number between 0.0 and 1.0.
+                """
+                title_response = llm.complete(title_prompt)
+                try:
+                    match = re.search(r'[0-9]*\.?[0-9]+', title_response.text)
+                    title_relevance = float(match.group()) if match else 0.3
+                    title_relevance = max(0.0, min(1.0, title_relevance))
+                except:
+                    title_relevance = 0.3
+
+                # Нормализация
+                normalized_vote_avg = vote_average / 10.0 if vote_average > 0 else 0.5
+                normalized_imdb = imdb_rating / 10.0 if imdb_rating > 0 else 0.5
+
+                # Формула
+                final_score = (
+                        semantic_score * 0.65 +
+                        narrative_similarity * 0.15 +
+                        genre_alignment * 0.04 +
+                        title_relevance * 0.04 +
+                        normalized_vote_avg * 0.02 +
+                        normalized_imdb * 0.10
+                )
+
+                return {
+                    "movie_title": movie_title,
+                    "final_score": round(final_score, 4),
+                    # movie_data добавим снаружи во wrapper-е
+                }
+
+            except Exception as e:
+                logger.error(f"Error evaluating {movie_data.get('title', 'Unknown')}: {e}")
+                return {"movie_title": "Error", "final_score": 0}
+
+        return FunctionTool.from_defaults(
+            fn=evaluate_movie_comprehensive,
+            name="comprehensive_evaluation",
+            description="Evaluate one movie using comprehensive weighted formula"
+        )
+
+    def _create_justification_tool(self) -> FunctionTool:
+        """Инструмент создания обоснований"""
+        llm = self.llm
+
+        def create_detailed_justification(user_query: str,
+                                          movie_data: dict,
+                                          evaluation_data: dict,
+                                          **kwargs) -> dict:
+            try:
+                if "title" not in movie_data and isinstance(movie_data, dict):
+                    movie_data = dict(next(iter(movie_data.values())))
+
+                movie_title = movie_data.get("title", "Unknown")
+                overview = movie_data.get("overview", "")[:220]
+                genres = movie_data.get("genres", "Unknown")
+                vote_average = movie_data.get('vote_average', 0)
+                imdb_rating = movie_data.get('imdb_rating', 0)
+
+                # ВАШ ОРИГИНАЛЬНЫЙ ПРОМПТ
+                justification_prompt = f"""
+                You are a seasoned film critic. Write an ENGLISH explanation (exactly 4-5
+sentences, one blank line, then a signature line).
+
+USER QUERY:
+"{user_query}"
+
+MOVIE DATA
+Title          : {movie_title}
+Genres         : {genres}
+Overview (cut) : {overview}
+TMDB / IMDb    : {vote_average}/10 • {imdb_rating}/10
+RelevanceScore : {evaluation_data.get('final_score', 0)}
+
+WRITING RULES
+1. Output **only the finished justification**.  
+2. NO planning words like "Next", "Then", "Need to", "Make sure", etc.  
+3. NO meta-instructions or bullet lists.  
+4. 1st-4th sentences must cover:
+   • direct plot / theme overlap  
+   • genre & narrative alignment  
+   • one unique shared element  
+   • (optionally) quality note via rating  
+5. After a single blank line add EXACTLY:
+
+"The relevance level of the film {movie_title} to your description is {evaluation_data.get('final_score', 0)}"
+"""
+
+                response = llm.complete(justification_prompt)
+                justification_text = response.text.strip()
+
+                return {
+                    "movie_title": movie_title,
+                    "justification": justification_text,
+                    "evaluation_score": evaluation_data.get('final_score', 0),
+                    "created_at": datetime.datetime.utcnow().isoformat() + "Z"
+                }
+
+            except Exception as e:
+                return {
+                    "movie_title": movie_data.get('title', 'Unknown'),
+                    "justification": f"Error creating justification: {str(e)}",
+                    "error": str(e)
+                }
+
+        return FunctionTool.from_defaults(
+            fn=create_detailed_justification,
+            name="create_justification",
+            description="Create detailed justification for movie recommendation"
+        )
+
+    # --- Вспомогательные методы (Точно как в old версии) ---
+
+    @staticmethod
+    def _extract_quality_notes(vote_average, imdb_rating):
+        notes = []
+        if vote_average >= 8.0: notes.append("Высокий рейтинг TMDB")
+        if imdb_rating >= 8.0: notes.append("Высокий рейтинг IMDb")
+        if vote_average >= 7.0 and imdb_rating >= 7.0: notes.append("Стабильно высокие оценки")
+        return notes
+
+    @staticmethod
+    def _safe_year(release_date) -> str:
+        from datetime import date, datetime
+        if isinstance(release_date, (date, datetime)): return str(release_date.year)
+        if isinstance(release_date, str) and len(release_date) >= 4: return release_date[:4]
+        return "Unknown"
+
+    @staticmethod
+    def _format_movie_card(movie_data: dict, justification: str, rank: int) -> str:
+        if "title" not in movie_data and isinstance(movie_data, dict):
+            movie_data = dict(next(iter(movie_data.values())))
+
+        title = movie_data.get('title', 'Unknown')
+        original_title = movie_data.get('original_title', '')
+        release_date = movie_data.get('release_date', 'Unknown')
+        year = ExpertAgent._safe_year(release_date)
+        overview = movie_data.get('overview', 'No overview available')
+        genres = movie_data.get('genres', 'Unknown')
+        tagline = movie_data.get('tagline', '')
+        vote_average = movie_data.get('vote_average', 0)
+        vote_count = movie_data.get('vote_count', 0)
+        imdb_rating = movie_data.get('imdb_rating', 0)
+        popularity = movie_data.get('popularity', 0)
+        runtime = movie_data.get('runtime', 0)
+        budget = movie_data.get('budget', 0)
+        revenue = movie_data.get('revenue', 0)
+        director = movie_data.get('director', 'Unknown')
+        cast = movie_data.get('cast', 'Unknown')
+
+        return f"""**{rank}. {title}** ({year})
+*{original_title}* {f'• {tagline}' if tagline else ''}
+
+**Genres:** {genres}
+
+**Overview:** {overview}
+
+**📊 Ratings:**
+• ⭐ TMDB: {vote_average}/10 ({vote_count:,} голосов)
+• 🎬 IMDb: {imdb_rating}/10
+• 📈 Popularity: {popularity:.0f}
+
+**🎥 Technical data:**
+• ⏱️ Runtime: {runtime} мин
+• 💰 Budget: ${budget:,} USD
+• 💵 Revenue: ${revenue:,} USD
+
+**👥 Cast:**
+• 🎬 Director: {director}
+• 🎭 Cast: {cast}
+
+**🎯 Justification:**
+{justification}"""

agents/modal_agents.py

@@ -0,0 +1,450 @@
+# agents/modal_agents.py
+import modal
+import os
+import logging
+import datetime
+
+# Настройка логирования
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+app = modal.App("movie-agents-nebius")
+
+# Образ с зависимостями для агентов
+agents_image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .pip_install(
+        "llama-index-core>=0.10.0",
+        "llama-index-llms-openai>=0.1.0",
+        "openai>=1.0.0",
+        "requests>=2.31.0"
+    )
+    # ✅ ДОБАВЛЕНО: Монтируем папки agents и evaluation
+    .add_local_dir("agents", remote_path="/root/agents")
+    .add_local_dir("evaluation", remote_path="/root/evaluation")
+)
+
+
+# --- СУЩЕСТВУЮЩИЕ АГЕНТЫ (без изменений) ---
+
+@app.function(
+    image=agents_image,
+    secrets=[modal.Secret.from_name("nebius-secret")],
+    timeout=300
+)
+def process_editor_agent(user_text: str, use_react: bool = False) -> dict:
+    """EditorAgent: улучшение текста"""
+    from agents.editor_agent import EditorAgent
+
+    nebius_api_key = os.environ.get("NEBIUS_API_KEY")
+    if not nebius_api_key:
+        raise ValueError("NEBIUS_API_KEY not found in Modal secrets")
+
+    editor = EditorAgent(nebius_api_key, use_react=use_react)
+    result = editor.process_and_improve_text(user_text)
+    logger.info(f"Editor result: {result}")
+    return result
+
+
+@app.function(
+    image=agents_image,
+    secrets=[modal.Secret.from_name("nebius-secret")],
+    timeout=300
+)
+def process_critic_agent(plot_description: str, action: str = "create", feedback: str = None) -> dict:
+    """FilmCriticAgent: создание синопсиса"""
+    from agents.critic_agent_nebius import FilmCriticAgent
+
+    nebius_api_key = os.environ.get("NEBIUS_API_KEY")
+    critic = FilmCriticAgent(nebius_api_key)
+
+    if action == "create":
+        result = critic.create_overview(plot_description)
+    elif action == "refine" and feedback:
+        result = critic.refine_with_feedback(plot_description, feedback)
+    else:
+        raise ValueError("Invalid action")
+
+    return result
+
+
+@app.function(
+    image=agents_image,
+    secrets=[modal.Secret.from_name("nebius-secret")],
+    timeout=300,
+    max_containers=1 # Макс 3 одновременных кодирования
+)
+def process_expert_agent(user_query: str, search_results: list) -> dict:
+    """ExpertAgent: финальный отбор"""
+    from agents.expert_agent import ExpertAgent
+
+    nebius_api_key = os.environ.get("NEBIUS_API_KEY")
+    expert = ExpertAgent(nebius_api_key)
+
+    return expert.analyze_and_recommend(user_query, search_results)
+
+
+# --- НОВЫЕ ФУНКЦИИ КООРДИНАТОРА ---
+
+@app.function(
+    image=agents_image,
+    secrets=[modal.Secret.from_name("nebius-secret")],
+    timeout=120,
+    volumes={"/data": modal.Volume.from_name("tmdb-data")}
+)
+def process_coordinator_check(user_text: str, attempts: int) -> dict:
+    """
+    1. Проверяет текст пользователя (длина, смысл).
+    Вызывается в начале пайплайна.
+    """
+    from agents.coordinator import CoordinatorAgent
+    import os
+
+    nebius_api_key = os.environ.get("NEBIUS_API_KEY")
+    agent = CoordinatorAgent(nebius_api_key)
+
+    # Вызываем метод analyze_input (который мы добавили в coordinator.py)
+    return agent.analyze_input(user_text, attempts)
+
+
+@app.function(
+    image=agents_image,
+    secrets=[modal.Secret.from_name("nebius-secret")],
+    timeout=60
+)
+def process_coordinator_suggestion(inputs: list, genre: str) -> dict:
+    """
+    2. Генерирует предложение истории (Romantic/Humorous).
+    Вызывается, если пользователь сделал несколько неудачных попыток.
+    """
+    from agents.coordinator import CoordinatorAgent
+    import os
+
+    nebius_api_key = os.environ.get("NEBIUS_API_KEY")
+    agent = CoordinatorAgent(nebius_api_key)
+
+    # Вызываем метод generate_suggestion
+    return agent.generate_suggestion(inputs, genre)
+
+
+@app.function(
+    image=agents_image,
+    secrets=[modal.Secret.from_name("nebius-secret")],
+    timeout=120,  # Оценка может занять время
+    volumes={"/data": modal.Volume.from_name("tmdb-data")}
+)
+def process_shadow_evaluation(
+        user_story: str,
+        expert_card: str,
+        movie_metadata: dict,
+        movie_rank: int = 1  # ✅ НОВЫЙ параметр: ранг фильма (1, 2, 3)
+):
+    """
+    Фоновая задача (AgentOps): запускает LLM-судью для оценки качества ответа.
+    Выполняется ПОСЛЕДОВАТЕЛЬНО для гарантии записи в Volume.
+    Результат пишется в логи (или в базу данных в будущем).
+    """
+    from evaluation.judges import PersuasionJudge
+    import os
+    import json
+    import time
+    import fcntl  # ✅ Для блокировки файла
+
+    nebius_api_key = os.environ.get("NEBIUS_API_KEY")
+    judge = PersuasionJudge(nebius_api_key)
+
+    movie_title = movie_metadata.get("title", "Unknown")
+
+    print(f"🕵️ Starting Shadow Eval for Rank #{movie_rank}: {movie_title}")
+
+    result = judge.evaluate_real_world_interaction(user_story, expert_card, movie_metadata)
+
+    # Логируем результат с указанием ранга
+    print(f"📊 SHADOW EVAL RESULT (Rank #{movie_rank}):")
+    print(f"   Movie: {movie_title}")
+    print(f"   Groundedness: {result.get('groundedness_score', 0)}")
+    print(f"   Coherence: {result.get('coherence_score', 0)}/5")
+    print(f"   Hallucination: {result.get('hallucination_detected', False)}")
+
+    if result.get("hallucination_detected"):
+        print(f"🚨 ALERT (Rank #{movie_rank}): {result.get('hallucination_details')}")
+
+    # Сохранение в JSONL файл на Volume
+    volume = modal.Volume.from_name("tmdb-data")
+
+    log_entry = {
+        "timestamp": datetime.datetime.utcnow().isoformat(),
+        "movie_rank": movie_rank,  # ✅ Добавлен ранг
+        "movie_title": movie_title,  # ✅ Добавлено название
+        "user_story": user_story[:200] + "...",  # Обрезаем для компактности
+        "eval_result": result
+    }
+
+    # ✅ Правильный путь к файлу в Volume
+    log_path = "/data/shadow_eval_logs.jsonl"
+
+    # ✅ Простая запись с commit после каждой записи
+    max_retries = 3
+    retry_delay = 1.0
+
+    # Аппендим строку в файл логов
+    for attempt in range(max_retries):
+        try:
+            # ✅ Reload volume перед записью
+            volume.reload()
+
+            # ✅ Открываем файл с блокировкой
+            with open(log_path, "a") as f:
+                # Блокируем файл на время записи
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                try:
+                    f.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
+                    f.flush()  # ✅ Принудительная запись на диск
+                    os.fsync(f.fileno())  # ✅ Гарантия записи
+                finally:
+                    fcntl.flock(f.fileno(), fcntl.LOCK_UN)  # Разблокируем
+
+            # ✅ Commit сразу после записи
+            volume.commit()
+            print(f"✅ Log written for Rank #{movie_rank}: {movie_title}")
+            break  # Успешно записали, выходим
+
+        except FileNotFoundError:
+            if attempt == 0:  # Создаём файл только при первой попытке
+                # Если файл не существует, создаём его
+                print(f"⚠️ Log file not found, creating {log_path}")
+                try:
+                    with open(log_path, "w") as f:
+                        fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                        f.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
+                        f.flush()
+                        os.fsync(f.fileno())
+                        fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+                    volume.commit()
+                    print(f"✅ Log file created and written for Rank #{movie_rank}")
+                    break
+                except Exception as create_error:
+                    print(f"❌ Failed to create log file: {create_error}")
+            time.sleep(retry_delay)
+
+        except Exception as e:
+            print(f"⚠️ Write attempt {attempt + 1}/{max_retries} failed: {e}")
+            if attempt < max_retries - 1:
+                time.sleep(retry_delay)
+            else:
+                print(f"❌ Failed to write log for Rank #{movie_rank} after {max_retries} attempts: {e}")
+
+    return result
+
+
+@app.function(
+    image=agents_image,
+    volumes={"/data": modal.Volume.from_name("tmdb-data")},
+    timeout=10
+)
+def check_daily_limit_remote(max_limit: int) -> bool:
+    """Проверяет лимит и инкрементирует счетчик на Volume"""
+    import json
+    import os
+    from datetime import datetime
+
+    stats_file = "/data/daily_usage_stats.json"
+    today = datetime.utcnow().strftime("%Y-%m-%d")
+
+    try:
+        data = {}
+        if os.path.exists(stats_file):
+            with open(stats_file, 'r') as f:
+                data = json.load(f)
+
+        current_count = data.get(today, 0)
+
+        if current_count >= max_limit:
+            print(f"🛑 Daily limit reached: {current_count}/{max_limit}")
+            return False
+
+        # Инкрементируем
+        data[today] = current_count + 1
+
+        # Очищаем старые даты (опционально, чтобы файл не раздувался)
+        if len(data) > 5:
+            data = {k: v for k, v in data.items() if k == today}
+
+        with open(stats_file, 'w') as f:
+            json.dump(data, f)
+
+        print(f"✅ Request allowed. Today: {current_count + 1}/{max_limit}")
+        return True
+
+    except Exception as e:
+        print(f"Error checking limit: {e}")
+        return True  # Fail open (разрешить, если ошибка доступа к файлу)
+
+# Подключаем тот же словарь
+active_users_dict = modal.Dict.from_name("cinematch-active-users", create_if_missing=True)
+
+@app.function(
+    image=agents_image,
+    timeout=30  # Очень быстрая функция
+)
+def try_acquire_slot(session_id: str, max_concurrent: int = 3) -> dict:
+    """
+    Пытается занять слот для пользователя.
+    Возвращает {'success': True} если место есть.
+    Возвращает {'success': False, 'message': ...} если занято.
+    """
+    import time
+
+    current_time = time.time()
+
+    # 1. Очистка "зомби" (тех, кто упал с ошибкой и не освободил слот)
+    # Если сессия висит больше 10 минут (600 сек) - удаляем её
+    keys_to_remove = []
+    for sid, timestamp in active_users_dict.items():
+        if current_time - timestamp > 600:
+            keys_to_remove.append(sid)
+
+    for k in keys_to_remove:
+        active_users_dict.pop(k)
+
+    # 2. Проверка: пользователь уже активен?
+    if session_id in active_users_dict:
+        # Обновляем timestamp (keep-alive)
+        active_users_dict[session_id] = current_time
+        return {"success": True}
+
+    # 3. Проверка свободных мест
+    current_active_count = len(list(active_users_dict.keys()))
+
+    if current_active_count >= max_concurrent:
+        return {
+            "success": False,
+            "message": "⚠️ **System Busy**\n\nToo many users are searching right now. "
+                       "To ensure high speed, we limit simultaneous searches.\n\n"
+                       "**Please try again in 2-3 minutes.**"
+        }
+
+    # 4. Занимаем место
+    active_users_dict[session_id] = current_time
+    print(f"✅ Slot acquired for {session_id}. "
+          f"Active: {len(list(active_users_dict.keys()))}/{max_concurrent}")
+    return {"success": True}
+
+@app.function(
+    image=agents_image,
+    timeout=10
+)
+def release_slot(session_id: str):
+    """Освобождает слот после завершения работы"""
+    if session_id in active_users_dict:
+        active_users_dict.pop(session_id)
+        print(f"🏁 Slot released for {session_id}. Remaining: {len(list(active_users_dict.keys()))}")
+
+
+@app.function(
+    image=agents_image,
+    secrets=[modal.Secret.from_name("nebius-secret")],
+    timeout=300,  # ✅ 5 минут ТОЛЬКО на обработку (без учёта очереди!)
+    volumes={"/data": modal.Volume.from_name("tmdb-data")}
+)
+def process_full_pipeline(session_id: str, user_text: str, session_data: dict) -> dict:
+    """
+    Выполняет полный пайплайн обработки ПОСЛЕ получения слота.
+    Timeout считается ТОЛЬКО с момента вызова этой функции.
+    """
+    from agents.editor_agent import EditorAgent
+    from agents.critic_agent_nebius import FilmCriticAgent
+    from agents.expert_agent import ExpertAgent
+    from agents.retriever import RetrieverAgent
+    from evaluation.judges import PersuasionJudge
+    import os
+
+    nebius_api_key = os.environ.get("NEBIUS_API_KEY")
+
+    try:
+        # 1. Editor Agent
+        editor = EditorAgent(nebius_api_key, use_react=False)
+        editor_result = editor.process_and_improve_text(user_text)
+
+        if not editor_result.get("approved", False):
+            return {
+                "status": "insufficient_length",
+                "message": editor_result.get("message", "Input quality too low"),
+                "original_plot": user_text
+            }
+
+        improved_text = editor_result.get("improved_text", user_text)
+
+        # 2. Critic Agent
+        critic = FilmCriticAgent(nebius_api_key)
+        overview_result = critic.create_overview(improved_text)
+        overview = overview_result.get("overview", str(improved_text))
+
+        # 3. Retriever (Search)
+        retriever = RetrieverAgent()
+        if not retriever.is_ready:
+            return {"status": "error", "message": "Search functions unavailable"}
+
+        retrieval_data = retriever.retrieve_candidates(overview, top_k=20, rerank_top_n=10)
+        candidates = retrieval_data.get("candidates", [])
+
+        if not candidates:
+            return {"status": "error", "message": "No movies found"}
+
+        # 4. Expert Agent
+        expert = ExpertAgent(nebius_api_key)
+        expert_result = expert.analyze_and_recommend(user_text, candidates)
+
+        recommendations = expert_result.get("explanations", "")
+        selected_movies = expert_result.get("selected_movies", [])
+
+        # 5. Shadow Evaluation (фоновые задачи)
+        try:
+            if selected_movies and len(selected_movies) > 0:
+                judge = PersuasionJudge(nebius_api_key)
+                eval_results = []
+
+                for i, movie in enumerate(selected_movies[:3], 1):
+                    movie_data = movie.get("movie_data", {})
+                    movie_title = movie_data.get("title", "Unknown")
+                    justification = movie.get("justification", "")
+                    full_overview = movie_data.get("overview", "No overview")
+
+                    expert_card_single = f"""
+                    **Recommendation #{i}: {movie_title}**
+                    **Genres:** {movie_data.get('genres', 'Unknown')}
+                    **Full Overview:** {full_overview}
+                    **Agent's Justification:**
+                    {justification}
+                    **Relevance Score:** {movie.get('evaluation', {}).get('final_score', 'N/A')}
+                    """
+
+                    # Синхронная оценка (т.к. уже внутри Modal функции)
+                    result = judge.evaluate_real_world_interaction(
+                        user_story=user_text,
+                        expert_card=expert_card_single,
+                        movie_metadata=movie_data
+                    )
+                    eval_results.append(result)
+
+                    # Сохраняем в Volume (упрощённо)
+                    print(f"📊 Shadow Eval #{i}: {movie_title} - {result}")
+
+        except Exception as e:
+            print(f"⚠️ Shadow eval failed: {e}")
+
+        # Возвращаем результат
+        return {
+            "status": "search_completed",
+            "original_plot": user_text,
+            "improved_plot": improved_text,
+            "movie_overview": overview,
+            "recommendations": recommendations,
+            "total_analyzed": len(candidates),
+            "performance_metrics": retrieval_data.get("metrics", {})
+        }
+
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+        return {"status": "error", "message": f"Pipeline error: {str(e)}"}

agents/modal_orchestrator.py

@@ -0,0 +1,482 @@
+# agents/modal_orchestrator.py
+import datetime
+import uuid
+import logging
+import modal
+import json
+
+# Импортируем агентов, включая новые функции для Координатора
+from agents.modal_agents import (
+    process_editor_agent,
+    process_critic_agent,
+    process_expert_agent,
+    process_coordinator_check,  # ✅ Новая функция проверки
+    process_coordinator_suggestion,  # ✅ Новая функция предложений
+    process_shadow_evaluation,  # ✅ NEW: Импорт функции оценки
+    process_full_pipeline,
+    try_acquire_slot,
+    release_slot
+)
+
+# Импортируем SessionStore и Retriever
+from memory.session_store import SessionStore
+from agents.retriever import RetrieverAgent
+
+from agents.modal_agents import try_acquire_slot, release_slot
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class ModalMovieSearchOrchestrator:
+    def __init__(self):
+
+        # Инициализируем Ретривера вместо прямых функций ---
+        self.retriever = RetrieverAgent()
+
+        if self.retriever.is_ready:
+            logger.info("✅ Connected to Modal functions via Retriever")
+            self.functions_available = True
+        else:
+            logger.error("❌ Retriever functions not found")
+            self.functions_available = False
+
+        # 2. Инициализация Session Store вместо словаря conversation_state
+        self.session_store = SessionStore()
+        # Создаем активную сессию (для однопользовательского режима -
+        # в многопользовательском app это передавалось бы в аргументах)
+        self.sid = self.session_store.create_session()
+
+    # def check_daily_limit(self) -> bool:
+    #     """Проверяет, не превышен ли лимит запросов на сегодня"""
+    #     import datetime
+    #
+    #     today = datetime.datetime.utcnow().strftime("%Y-%m-%d")
+    #     limit_file = "/data/usage_stats.json"  # Храним на Volume
+    #     DAILY_LIMIT = 50  # Например, 50 диалогов в день
+    #
+    #     try:
+    #         # Читаем статистику (псевдокод, нужны импорты json/os)
+    #         if os.path.exists(limit_file):
+    #             with open(limit_file, 'r') as f:
+    #                 stats = json.load(f)
+    #         else:
+    #             stats = {}
+    #
+    #         current_count = stats.get(today, 0)
+    #
+    #         if current_count >= DAILY_LIMIT:
+    #             return False  # Лимит исчерпан
+    #
+    #         # Увеличиваем счетчик (в реальном коде нужно делать это после успешного запуска)
+    #         # Но для простоты можно и здесь
+    #         # stats[today] = current_count + 1
+    #         # with open(limit_file, 'w') as f:
+    #         #    json.dump(stats, f)
+    #
+    #         return True
+    #
+    #     except Exception:
+    #         return True  # Если ошибка чтения, лучше пропустить, чем блокировать
+
+    def check_and_update_daily_limit(self, max_daily_requests=50) -> bool:
+        """
+        Проверяет и обновляет счетчик запросов на Modal Volume.
+        Возвращает True, если лимит НЕ превышен (можно работать).
+        Возвращает False, если лимит исчерпан.
+        """
+        # Путь к файлу на Volume (он сохраняется между перезапусками)
+        # Убедитесь, что Volume '/data' примонтирован к функции, которая вызывает этот метод!
+        # В вашем случае process_coordinator_check имеет volume, но оркестратор работает ЛОКАЛЬНО (в app.py).
+
+        # ВАЖНО: Поскольку Orchestrator работает в app.py (на HuggingFace),
+        # он НЕ ИМЕЕТ доступа к Modal Volume напрямую через 'open()'.
+        # Нам нужно вынести эту проверку в Modal-функцию.
+
+        # Поэтому мы просто вызовем специальную легкую функцию проверки на Modal.
+        try:
+            # Импортируем функцию проверки (ее нужно добавить в modal_agents.py)
+            from agents.modal_agents import check_daily_limit_remote
+
+            # Вызываем удаленно
+            is_allowed = check_daily_limit_remote.remote(max_limit=max_daily_requests)
+            return is_allowed
+        except Exception as e:
+            logger.error(f"Failed to check limit: {e}")
+            return True  # В случае ошибки лучше пропустить, чем блокировать всех
+
+    async def process_user_input(self, user_text: str) -> dict:
+        """
+        Главный цикл обработки.
+        Логика: 1/0/2 Menu -> Custom Mode -> Suggestions -> Pipeline
+        """
+        logger.info(f"Processing input: {user_text}")
+
+        # # Сохраняем текущий ввод в общий контекст
+        self.session_store.add_user_input(self.sid, user_text)
+
+        # Получаем актуальное состояние
+        session = self.session_store.get_session(self.sid)
+
+        # 1. СНАЧАЛА ПРОВЕРЯЕМ ЛИМИТ (до запуска тяжелых агентов)
+        if not self.check_and_update_daily_limit(max_daily_requests=50):
+            return {
+                "status": "error",
+                "message": "⚠️ **Daily Demo Limit Reached**\n\nTo control costs during the Hackathon, we have a daily limit on requests. Please come back tomorrow!\n\n*(Reset time: 00:00 UTC)*"
+            }
+        # ------------------------------------------------------------------
+        # ЭТАП 1: Обработка ответа на предложение (1/0/2)
+        # ------------------------------------------------------------------
+        if session.get("suggested_plot"):
+            # Проверяем ответ пользователя
+            user_input_cleaned = user_text.strip().lower()
+            # is_agreement = any(word in user_text.lower() for word in ["yes", "sure", "ok", "agree", "да", "конечно"])
+
+            # ✅ Опция 1: Согласие с предложением
+            if user_input_cleaned == "1":
+                # Пользователь согласился
+                logger.info("User accepted suggestion (input: 1)")
+                user_text = session["suggested_plot"]
+
+                self.session_store.update_state(self.sid, "suggested_plot", None)
+                self.session_store.reset_attempts(self.sid)
+                self.session_store.set_custom_mode(self.sid, False)
+                # Продолжаем с user_text = предложенный сюжет
+
+            # ✅ Опция 0: Отказ от предложения
+            elif user_input_cleaned == "0":
+                logger.info("User rejected suggestion (input: 0)")
+
+                self.session_store.update_state(self.sid, "suggested_plot", None)
+                self.session_store.increment_attempts(self.sid)
+                self.session_store.set_custom_mode(self.sid, False)
+                # Переходим к следующему этапу (романтика/юмор/выход - Этап 2)
+
+            # ✅ Опция 2: Пользователь хочет ввести свой сюжет
+            elif user_input_cleaned == "2":
+                logger.info("User chose to provide custom plot (input: 2)")
+
+                self.session_store.update_state(self.sid, "suggested_plot", None)
+                self.session_store.set_custom_mode(self.sid, True)  # Включаем строгий режим
+                self.session_store.reset_attempts(self.sid)
+
+                msg = (
+                    "**📝 Custom Plot Entry Mode**\n\n"
+                    "Please provide your own movie plot description following these **strict requirements**:\n\n"
+                    "**✅ Requirements:**\n"
+                    "• **Minimum 50 words** (approximately 3-5 sentences)\n"
+                    "• **Clear plot structure** with characters, conflict, and setting\n"
+                    "• **English language only**\n"
+                    "• **Proper grammar** and coherent narrative\n\n"
+                    "**⚠️ IMPORTANT:**\n"
+                    "• You have **ONE attempt** only\n"
+                    "• If your description doesn't meet requirements, the session will end gracefully\n"
+                    "• The system will validate your input strictly\n\n"
+                    "**💡 Example of good description (56 words):**\n"
+                    "_\"In a dystopian future where water is scarce, a young rebel discovers a hidden map "
+                    "leading to an underground ocean. Along with a cynical smuggler and a defected soldier, "
+                    "she must cross the scorched wasteland, chased by the warlord's army who wants to control "
+                    "the water supply for themselves. They face sandstorms, betrayal, and mechanical beasts.\"_\n\n"
+                    "**❌ Example of BAD description (too short - 12 words):**\n\n"
+                    "_\"A girl finds water in the desert and fights bad guys.\"_\n\n"
+                    "**Please enter your plot description now:**"
+                )
+
+                # self._update_history(user_text, msg)
+                self.session_store.add_history(self.sid, user_text, msg)
+                return {
+                    "status": "awaiting_custom_plot",
+                    "message": msg,
+                    "custom_mode": True
+                }
+
+            else:
+                # ✅ ЛЮБОЙ другой ввод = согласие (по умолчанию)
+                logger.info(f"User provided text instead of 1/0/2, treating as custom plot: {user_text[:50]}...")
+
+                self.session_store.update_state(self.sid, "suggested_plot", None)
+                self.session_store.set_custom_mode(self.sid, True)
+                self.session_store.reset_attempts(self.sid)
+                # НЕ возвращаемся, продолжаем обработку user_text как custom plot
+
+                # Обновляем локальную переменную session
+                session = self.session_store.get_session(self.sid)
+
+        # ------------------------------------------------------------------
+        # ЭТАП 2: Логика предложений (Suggestions) на основе попыток
+        # ------------------------------------------------------------------
+        attempts = session.get("attempts", 0)
+        user_inputs = session.get("user_inputs", [])
+
+        # Попытка 3 (было 2 неудачи) -> Предлагаем Романтику
+        # if session["attempts"] == 2:
+        if attempts == 2:
+            logger.info("Attempt 2 failed, suggesting Romantic plot")
+
+            suggestion = await process_coordinator_suggestion.remote.aio(user_inputs, "romantic")
+
+            self.session_store.update_state(self.sid, "suggested_plot", suggestion["suggested_story"])
+
+            msg = (
+                f"{suggestion['message']}\n\n"
+                f"**Proposed Plot (Romantic):**\n"
+                f"_{suggestion['suggested_story']}_\n\n"
+                f"{'=' * 60}\n"
+                f"**📍 Please choose an option:**\n"
+                f"• Press **1** to accept this plot (YES)\n"
+                f"• Press **0** to try a different genre (NO)\n"
+                f"• Press **2** to provide your own plot description\n\n"
+                f"⚠️ **Note:** Any other input will be treated as your custom plot description "
+                f"(same as pressing 2)."
+            )
+
+            self.session_store.add_history(self.sid, user_text, msg)
+            return {
+                "status": "suggestion",
+                "message": msg,
+                "is_suggestion": True,
+                "suggestion_type": "romantic"
+            }
+
+        # Попытка 4 (отказ от романтики) -> Предлагаем Юмор
+        if attempts == 3:
+            logger.info("Romantic rejected, suggesting Humorous plot")
+
+            suggestion = await process_coordinator_suggestion.remote.aio(user_inputs, "humorous")
+
+            self.session_store.update_state(self.sid, "suggested_plot", suggestion["suggested_story"])
+
+            msg = (
+                f"{suggestion['message']}\n\n"
+                f"**Proposed Plot (Humorous):**\n"
+                f"_{suggestion['suggested_story']}_\n\n"
+                f"{'=' * 60}\n"
+                f"**📍 Please choose an option:**\n"
+                f"• Press **1** to accept this plot (YES)\n"
+                f"• Press **0** to end session (NO)\n"
+                f"• Press **2** to provide your own plot description\n\n"
+                f"⚠️ **Note:** Any other input will be treated as your custom plot description "
+                f"(same as pressing 2)."
+            )
+
+            self.session_store.add_history(self.sid, user_text, msg)
+            return {
+                "status": "suggestion",
+                "message": msg,
+                "is_suggestion": True,
+                "suggestion_type": "humorous"
+            }
+
+        # Попытка 5 (отказ от всего) -> Выход
+        if attempts >= 4:
+            logger.info("All suggestions rejected. Ending session.")
+            msg = (
+                "**👋 Session Ended**\n\n"
+                "It seems we can't find the right story today. "
+                "Please come back when you have a new idea! Have a great day!\n\n"
+                "**🔄 Ready to start fresh!** Feel free to describe a new movie plot anytime."
+            )
+            self.session_store.add_history(self.sid, user_text, msg)
+
+            return {
+                "status": "end_session",
+                "message": msg,
+                "end_session": True
+            }
+
+        # ------------------------------------------------------------------
+        # ЭТАП 3: Строгая проверка для Custom Plot (если флаг установлен)
+        # ------------------------------------------------------------------
+        # Выполняется, если это не ответ на предложение (или если предложение принято)
+        if session.get("custom_plot_mode", False):
+            logger.info(f"Custom plot mode: strict validation for: {user_text[:50]}...")
+
+            # Дополнительная проверка длины
+            word_count = len(user_text.split())
+
+            if word_count < 50:
+                msg = (
+                    f"**❌ Too Short**\n\n"
+                    f"Your description has only **{word_count} words**, but we need **minimum 50 words**.\n\n"
+                    f"{'=' * 60}\n"
+                    "**📋 Requirements:**\n"
+                    "• Minimum **50 words** (3-5 sentences)\n"
+                    "• Clear plot with characters and conflict\n"
+                    "• English language only\n"
+                    "• Proper grammar\n\n"
+                    f"{'=' * 60}\n"
+                    "**👋 Session Ended Gracefully**\n\n"
+                    "Don't worry! Take your time to craft a detailed plot description.\n"
+                    "When you're ready with a **complete story** (50+ words), feel free to start a new session.\n\n"
+                    "**💡 Tip:** Think about:\n"
+                    "• Who are the main characters?\n"
+                    "• What conflict do they face?\n"
+                    "• What's at stake?\n"
+                    "• What makes the story unique?\n\n"
+                    "**See you soon!** 🎬"
+                )
+
+                self.session_store.add_history(self.sid, user_text, msg)
+                self.session_store.set_custom_mode(self.sid, False)
+
+                return {
+                    "status": "custom_plot_too_short",
+                    "message": msg,
+                    "end_session": True
+                }
+
+            try:
+                # ✅ СТРОГАЯ проверка координатором
+                analysis = await process_coordinator_check.remote.aio(user_text, attempts)
+
+                if analysis["status"] == "insufficient":
+                    # ❌ Не прошло проверку в строгом режиме -> Благожелательный выход
+                    logger.warning(f"Custom plot rejected: {analysis.get('message')}")
+
+                    msg = (
+                        "**❌ Custom Plot Validation Failed**\n\n"
+                        f"{analysis.get('message', 'Your plot description does not meet the requirements.')}\n\n"
+                        "{'=' * 60}\n"
+                        "**📋 Requirements reminder:**\n"
+                        "• Minimum **50 words** (3-5 sentences)\n"
+                        "• Clear plot with characters and conflict\n"
+                        "• English language only\n"
+                        "• Proper grammar\n\n"
+                        "{'=' * 60}\n"
+                        "**👋 Session Ended Gracefully**\n\n"
+                        "Don't worry! Take your time to craft a detailed plot description.\n"
+                        "When you're ready with a **complete story** (50+ words), feel free to start a new session.\n\n"
+                        "**💡 Tip:** Think about:\n"
+                        "• Who are the main characters?\n"
+                        "• What conflict do they face?\n"
+                        "• What's at stake?\n"
+                        "• What makes the story unique?\n\n"
+                        "**See you soon!** 🎬"
+                    )
+
+                    self.session_store.add_history(self.sid, user_text, msg)
+                    self.session_store.set_custom_mode(self.sid, False)
+                    self.session_store.reset_attempts(self.sid)
+
+                    return {
+                        "status": "custom_plot_rejected",
+                        "message": msg,
+                        "end_session": True
+                    }
+
+                # ✅ Прошло строгую проверку -> продолжаем поиск
+                logger.info("Custom plot validated successfully!")
+
+                self.session_store.set_custom_mode(self.sid, False)
+                self.session_store.reset_attempts(self.sid)
+                # Продолжаем к ЭТАПУ 5 (поиск)
+
+            except Exception as e:
+                logger.error(f"Coordinator check failed in custom mode: {e}")
+                # Fallback: Если координатор упал, тоже делаем благожелательный выход
+                msg = (
+                    "**⚠️ System Error**\n\n"
+                    "We encountered an issue validating your plot. "
+                    "Please try again later with a detailed description.\n\n"
+                    "**See you soon!** 🎬"
+                )
+
+                self.session_store.set_custom_mode(self.sid, False)
+
+                return {
+                    "status": "error",
+                    "message": msg,
+                    "end_session": True
+                }
+
+        # ------------------------------------------------------------------
+        # ЭТАП 4: Стандартная проверка Координатором (для обычного режима).
+        # Пайплайн Поиска (Editor -> Critic -> Search -> Expert)
+        # ------------------------------------------------------------------
+        else:
+            try:
+                analysis = await process_coordinator_check.remote.aio(user_text, attempts)
+
+                if analysis["status"] == "insufficient":
+                    self.session_store.increment_attempts(self.sid)
+                    attempts_left = 2 - attempts  # (0->1 left, 1->0 left)
+
+                    warning_msg = analysis.get("message", "Please add more details.")
+                    if attempts_left > 0:
+                        warning_msg += f"\n\n_You have {attempts_left} attempt(s) left before I suggest a plot._"
+
+                    self.session_store.add_history(self.sid, user_text, warning_msg)
+
+                    return {
+                        "status": "insufficient_length",
+                        "message": warning_msg,
+                        "original_plot": user_text
+                    }
+
+                # Если статус 'valid' -> переходим к поиску
+                logger.info("Coordinator validated input. Starting pipeline.")
+
+                self.session_store.reset_attempts(self.sid)
+
+
+            except Exception as e:
+                logger.error(f"Coordinator check failed: {e}. Falling back to search.")
+
+        # ------------------------------------------------------------------
+        # ЭТАП 5: Пайплайн Поиска (Editor -> Critic -> Search -> Expert)
+        # ------------------------------------------------------------------
+        # 1. ПОПЫТКА ЗАНЯТЬ СЛОТ
+        # Используем session_id как уникальный ключ
+        slot_result = await try_acquire_slot.remote.aio(
+            session_id=self.sid,
+            max_concurrent=1  # <-- ЛИМИТ ЗДЕСЬ (3)
+        )
+
+        if not slot_result.get("success"):
+            # ВАЖНО: не держим запрос в Modal, говорим клиенту "system busy"
+            return {
+                "status": "busy",
+                "message": slot_result.get(
+                    "message",
+                    "⚠️ **System Busy**\n\nToo many users are searching right now. "
+                    "Please try again in 2–3 minutes."
+                ),
+                "retry_suggested": True
+            }
+
+        # ✅ ШАГ 2: СЛОТ ПОЛУЧЕН → ЗАПУСКАЕМ ОБРАБОТКУ (timeout=300s)
+        try:
+            # Теперь вызываем обработку в отдельной функции с НОВЫМ timeout
+            result = await process_full_pipeline.remote.aio(
+                session_id=self.sid,
+                user_text=user_text,
+                session_data=session
+            )
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Pipeline error: {e}")
+            return {"status": "error", "message": f"System error: {e}"}
+
+        finally:
+            # 2. ОСВОБОЖДЕНИЕ СЛОТА (Выполнится всегда, даже при ошибке)
+            # Запускаем в фоновом режиме ("fire and forget"), чтобы не задерживать ответ пользователю
+            release_slot.spawn(self.sid)
+
+    def get_conversation_summary(self):
+        """Возвращает текущее состояние для UI"""
+        session = self.session_store.get_session(self.sid)
+        if not session: return {}
+        return {
+            "session_id": session.get("session_id"),
+            "current_step": "completed" if session.get("final_recommendations") else "processing",
+            "attempts": session.get("attempts", 0),
+            "has_plot": bool(session.get("original_plot")),
+            "has_recommendations": bool(session.get("final_recommendations")),
+            "total_search_results": len(session.get("search_results", []))
+        }
+
+    def reset_conversation(self):
+        """Сброс сессии"""
+        self.session_store.clear_session_data(self.sid)

agents/nebius_simple.py

@@ -0,0 +1,109 @@
+# agents/nebius_simple.py
+
+import os
+from openai import OpenAI
+from typing import Any, Optional
+from llama_index.core.llms import CustomLLM, CompletionResponse, LLMMetadata
+from llama_index.core.llms.callbacks import llm_completion_callback
+from pydantic import Field, PrivateAttr
+
+
+class SimpleNebiusLLM(CustomLLM):
+    """Простой Nebius LLM через прямой OpenAI клиент"""
+
+    # ✅ ИСПРАВЛЕНО: Объявляем поля как Pydantic поля
+    api_key: str = Field(description="API ключ Nebius")
+    model_name: str = Field(default="meta-llama/Llama-3.3-70B-Instruct", description="Имя модели")
+    temperature: float = Field(default=0.3, description="Температура генерации")
+    base_url: str = Field(default="https://api.studio.nebius.com/v1/", description="Base URL для API")
+
+    # ✅ ИСПРАВЛЕНО: Используем PrivateAttr для OpenAI клиента
+    _client: OpenAI = PrivateAttr()
+
+    def __init__(
+            self,
+            api_key: str = None,
+            model: str = "meta-llama/Llama-3.3-70B-Instruct",
+            temperature: float = 0.3,
+            **kwargs
+    ):
+        api_key = api_key or os.environ.get("NEBIUS_API_KEY")
+        if not api_key:
+            raise ValueError("NEBIUS_API_KEY не найден")
+
+        # ✅ ИСПРАВЛЕНО: Инициализируем родительский класс с полями
+        super().__init__(
+            api_key=api_key,
+            model_name=model,
+            temperature=temperature,
+            **kwargs
+        )
+
+        # ✅ ИСПРАВЛЕНО: Создаем клиента как приватный атрибут
+        self._client = OpenAI(
+            base_url=self.base_url,
+            api_key=self.api_key
+        )
+
+    @property
+    def metadata(self) -> LLMMetadata:
+        return LLMMetadata(
+            context_window=128000,
+            num_output=4096,
+            model_name=self.model_name,
+        )
+
+    @llm_completion_callback()
+    def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
+        """Синхронное completion через Nebius API"""
+        try:
+            # ✅ ИСПРАВЛЕНО: Используем self._client
+            response = self._client.chat.completions.create(
+                model=self.model_name,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=self.temperature,
+                max_tokens=4096
+            )
+
+            text = response.choices[0].message.content
+            return CompletionResponse(text=text)
+
+        except Exception as e:
+            error_text = f"Nebius API error: {str(e)}"
+            return CompletionResponse(text=error_text)
+
+    @llm_completion_callback()
+    def stream_complete(self, prompt: str, **kwargs: Any):
+        """Потоковое completion"""
+        try:
+            response = self._client.chat.completions.create(
+                model=self.model_name,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=self.temperature,
+                max_tokens=4096,
+                stream=True
+            )
+
+            accumulated_text = ""
+            for chunk in response:
+                if chunk.choices[0].delta.content:
+                    content = chunk.choices[0].delta.content
+                    accumulated_text += content
+                    yield CompletionResponse(text=accumulated_text, delta=content)
+
+        except Exception as e:
+            error_text = f"Nebius streaming error: {str(e)}"
+            yield CompletionResponse(text=error_text, delta=error_text)
+
+
+def create_nebius_llm(
+        api_key: str = None,
+        model: str = "meta-llama/Llama-3.3-70B-Instruct",
+        temperature: float = 0.3
+) -> SimpleNebiusLLM:
+    """Фабричная функция для создания Nebius LLM"""
+    return SimpleNebiusLLM(
+        api_key=api_key,
+        model=model,
+        temperature=temperature
+    )

agents/retriever.py

@@ -0,0 +1,50 @@
+# agents/retriever.py
+import logging
+import modal
+from typing import List, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+
+class RetrieverAgent:
+    """
+    Агент-специалист по поиску.
+    Инкапсулирует логику: Text -> Embedding -> Vector Search -> Candidates.
+    """
+
+    def __init__(self):
+        try:
+            self.encode_func = modal.Function.from_name("tmdb-project", "encode_user_query")
+            self.search_func = modal.Function.from_name("tmdb-project", "search_similar_movies")
+            logger.info("✅ RetrieverAgent connected to Modal functions")
+            self.is_ready = True
+        except Exception as e:
+            logger.error(f"❌ RetrieverAgent failed to connect: {e}")
+            self.is_ready = False
+
+    def retrieve_candidates(self, overview_text: str, top_k: int = 20, rerank_top_n: int = 10) -> dict:
+        """
+        Возвращает словарь с кандидатами и метриками.
+        """
+        if not self.is_ready:
+            return {"error": "Search functions unavailable", "candidates": []}
+
+        try:
+            # 1. Кодирование (remove_entities=True для поиска по смыслу)
+            encoding_result = self.encode_func.remote(overview_text, remove_entities=True)
+
+            # 2. Поиск
+            search_results = self.search_func.remote(
+                query_embedding=encoding_result["embedding"],
+                query_narrative_features=encoding_result["narrative_features"],
+                top_k=top_k,
+                rerank_top_n=rerank_top_n
+            )
+
+            return {
+                "candidates": search_results.get("results", []),
+                "metrics": search_results.get("performance_metrics", {})
+            }
+        except Exception as e:
+            logger.error(f"Retrieval error: {e}")
+            return {"error": str(e), "candidates": []}

app.py

@@ -0,0 +1,359 @@
+# app_simplified.py
+import modal
+import gradio as gr
+import asyncio
+import os
+import logging
+
+# from modal_app import app
+# from agents.orchestrator import SimplifiedMovieSearchOrchestrator
+
+# Импорт Modal оркестратора вместо локального
+from agents.modal_orchestrator import ModalMovieSearchOrchestrator
+
+app = modal.App("movie-plot-search")
+
+# print("Trying to lookup functions...")
+# try:
+#     encode_func = modal.Function.from_name("tmdb-project", "encode_user_query")
+#     print("✅ encode_user_query function found")
+# except Exception as e:
+#     print(f"❌ Error looking up encode_user_query: {e}")
+
+
+# --- Основная функция запуска приложения ---
+def _run_main_app():
+    # Настройка логирования
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+
+    # Инициализация Modal оркестратора (без API ключа)
+    orchestrator = ModalMovieSearchOrchestrator()
+
+    async def chat_interface(message: str, history: list) -> tuple:
+        """ Основной интерфейс чата с Modal агентами + Nebius LLM5(только английский)"""
+        try:
+            logger.info(f"Processing user message: {message[:50]}...")
+
+            # ВСЕ LLM ВЫЗОВЫ ПРОИСХОДЯТ НА MODAL С NEBIUS API
+            result = await orchestrator.process_user_input(message)
+            logger.info(f"RESULT: {result}")
+            # Формирование ответа
+            response_parts = []
+            status = result.get("status")
+
+            # ---------- 1. Обработка статусов от оркестратора ----------
+            # ✅ Обработка случая недостаточной длины
+            # if result.get("status") == "insufficient_length":
+            #     response_parts.append("**❌ Text Too Short**")
+            #     response_parts.append(result.get("message", ""))
+
+            if status == "insufficient_length":
+                response_parts += [
+                    "**❗ **Editor Feedback:**",
+                    result.get("message", ""),
+                    "\n---\nPlot description is too short (min 50 words). "
+                    "Please expand your plot description and try again."
+                ]
+
+            # (2) Полный успех: найдено 3 фильма + экспертный отчёт
+            elif status == "search_completed":
+                logger.info(f"**✅ Поиск завершен! Найдены рекомендации фильмов**")
+                response_parts.append("**✅ Plot processed and search completed!**")
+                # ✅ Показываем improved plot для информации
+
+                if (result.get("improved_plot") and
+                        result.get("improved_plot") != result.get("original_plot")):
+                    logger.info(f"**📝 Улучшенное описание:** {result.get('improved_plot')}")
+                    response_parts.append(f"**📝 Improved plot:** {result.get('improved_plot')}")
+                # ✅ Показываем movie overview для информации
+
+                if result.get("movie_overview"):
+                    response_parts.append(f"\n**🎬 Generated movie overview:**\n"
+                                          f"{result.get('movie_overview')}")
+
+                # ✅ Основной блок рекомендаций с новым форматом
+                response_parts.append("\n" + "=" * 60)
+                response_parts.append("**🎯 EXPERT SYSTEM RECOMMENDATIONS**")
+                response_parts.append("=" * 60)
+
+                # response_parts.append(result.get("recommendations", ""))
+                recommendations = result.get("recommendations", "")
+
+                if isinstance(recommendations, dict):
+                    logger.warning("Received dict instead of string for recommendations")
+                    recommendations = recommendations.get("explanations", str(recommendations))
+
+                if isinstance(recommendations, str) and recommendations:
+                    response_parts.append(recommendations)
+                else:
+                    logger.error(f"Invalid recommendations type: {type(recommendations)}")
+                    response_parts.append("No recommendations were generated. Please try again.")
+
+                # if recommendations:
+                #     response_parts.append(recommendations)
+                # else:
+                #     response_parts.append("No recommendations were generated.")
+
+                # ✅ Метрики производительности
+                response_parts.append("\n" + "=" * 60)
+                response_parts.append("**📊 PERFORMANCE METRICS**")
+                response_parts.append("=" * 60)
+
+                metrics = result.get("performance_metrics", {})
+                if metrics:
+                    response_parts.append(f"🚀 **GPU Used:** {'✅ Yes' if metrics.get('using_gpu', False) else '❌ No'}")
+                    response_parts.append(f"⚡ **Search Time:** {metrics.get('search_time', 0):.3f}s")
+                    response_parts.append(f"🔄 **Total Processing Time:** {metrics.get('total_time', 0):.3f}s")
+                    response_parts.append(f"🎬 **Movies Analyzed:** {result.get('total_analyzed', 0)}")
+
+                    response_parts.append("\n" + "=" * 60)
+                    response_parts.append("**🔄 Ready for the next search!**")
+                    response_parts.append("Type a new movie plot and I will find more recommendations.")
+
+            elif status == "suggestion":
+                response_parts.append("**💡 AI Plot Suggestion**")
+                response_parts.append(result.get("message", ""))
+
+            elif status == "awaiting_custom_plot":
+                response_parts.append("**📝 Custom Plot Mode Activated**")
+                response_parts.append(result.get("message", ""))
+
+            # ✅ ДОБАВЛЕНО: Обработка выхода при коротком custom plot
+            elif status == "custom_plot_too_short":
+                # response_parts.append("**❌ Custom Plot Too Short**")
+                response_parts.append(result.get("message", "Your plot is too short."))
+
+            # ✅ ДОБАВЛЕНО: Обработка выхода при отклонении custom plot
+            elif status == "custom_plot_rejected":
+                # response_parts.append("**❌ Custom Plot Rejected**")
+                response_parts.append(result.get("message", "Your plot doesn't meet requirements."))
+
+            #         if result.get('methodology'):
+            #             response_parts.append(f"🧮 **Methodology:** {result.get('methodology')}")
+            #         if result.get('evaluation_formula'):
+            #             response_parts.append(f"📐 **Evaluation Formula:** {result.get('evaluation_formula')}")
+            #
+            #         # Russian comment: приглашение к новому поиску
+            #         response_parts.append("\n" + "=" * 30)
+            #         response_parts.append("**🔄 Ready for the next search!**")
+            #         response_parts.append("Type a new movie plot and I will find more recommendations.")
+            #
+            # # ✅ ДОБАВЛЕНО: Обработка статуса "suggestion"
+            # elif status == "suggestion":
+            #     response_parts.append("**💡 AI Plot Suggestion**")
+            #     response_parts.append(result.get("message", ""))
+            #     if result.get("is_suggestion"):
+            #         response_parts.append(
+            #             "\n**Would you like to proceed with this plot?** (Reply 'Yes' or provide your own)")
+
+            # ✅ ДОБАВЛЕНО: Обработка завершения сессии
+            elif status == "end_session":
+                # response_parts.append("**👋 Session Ended**")
+                response_parts.append(result.get("message", "Thank you for using Movie Plot Search!"))
+                # response_parts.append("\n---")
+                # response_parts.append("**🔄 Ready to start fresh!** Feel free to describe a new movie plot anytime.")
+
+            # ✅ Обработка ошибок
+            elif status == "error":
+                response_parts.append("**❌ System Error occurred:**")
+                response_parts.append(result.get("message", "Unknown error"))
+            else:
+                response_parts.append(f"⚠️ Unhandled status: {status}")
+                response_parts.append(f"Result details: {result}")  # добавлено
+                logger.warning(f"Unhandled status encountered: {status}")  # добавлено
+
+                # ---------- 2. Формируем ответ ВСЕГДА и историю (вне if блоков) ----------
+            assistant_reply = "\n".join(response_parts)
+            new_history = history + [
+                {"role": "user", "content": message},
+                {"role": "assistant", "content": assistant_reply}
+            ]
+
+            # ✅ КРИТИЧНО: return ВСЕГДА выполняется (вне if блоков)
+            return new_history, ""
+
+            # # Автообновление session info после обработки
+            # if status in ["search_completed", "needs_improvement", "insufficient_length"]:
+            #     # Обновляем session info автоматически
+            #     _ = get_session_info()  # обновит компонент через .then() в Gradio
+            #     logger.info(f"Session info updated: {_}")
+            #
+            #     # Очищаем поле ввода
+            #     return new_history, ""
+
+        except Exception as e:
+            logger.error(f"Error in chat interface: {e}")
+
+            # Формат messages для ошибок
+            # Обработка ошибок
+            new_history = history + [
+                {"role": "user", "content": message},
+                {"role": "assistant", "content": f"**❌ Unexpected error:** {e}"}
+            ]
+
+            return new_history, ""
+
+    def reset_chat():
+        """Сброс чата с логированием"""
+        logger.info("Resetting chat session")
+        orchestrator.reset_conversation()
+        return [], ""       # Возвращаем пустую историю
+
+    def get_session_info():
+        """Получение информации о текущей сессии"""
+        try:
+            # logger.warning(f"Summary type: {type(orchestrator.get_conversation_summary())} _
+            # Summary: {orchestrator.get_conversation_summary()}")
+            summary = orchestrator.get_conversation_summary()
+            logger.info(f"Getting session summary: {summary}")
+
+            return f"""**Hybrid Session Info:**
+    - ID: {summary['session_id']}
+    - Step: {summary['current_step']}
+    - Has Plot: {'✅' if summary.get('has_plot', False) else '❌'}
+    - Has Overview: {'✅' if summary.get('has_overview', False) else '❌'}
+    - Has Recommendations: {'✅' if summary.get('has_recommendations', False) else '❌'}
+    - Total Results: {summary.get('total_search_results', 0)}
+            """
+
+        except Exception as e:
+            logger.error(f"Error in get_session_info: {e}")
+            return f"Error getting session info: {e}"
+
+    def force_refresh_session_info():
+        """Принудительное обновление с логированием состояния"""
+        try:
+            # ✅ Дополнительное логирование для отладки
+            logger.info("Force refreshing session info...")
+            logger.info(f"Current orchestrator state: {orchestrator.conversation_state}")
+
+            summary = orchestrator.get_conversation_summary()
+            logger.info(f"Retrieved summary: {summary}")
+
+            return get_session_info()
+        except Exception as e:
+            logger.error(f"Error in force refresh: {e}")
+            return f"Refresh error: {e}"
+
+    # Создание интерфейса Gradio
+    with gr.Blocks(title="🎬 Movie Plot Search", theme=gr.themes.Soft()) as demo:
+        gr.Markdown("""
+            # 🎬 Movie Plot Search Engine
+            
+            **🏗️ Architecture:**
+                 🖥️ **UI**: Local Gradio interface;
+                 ⚡ **Agents**: Running on Modal Cloud;
+                 🤖 **LLM**: Nebius AI Studio API (Llama-3.3-70B-Instruct).
+
+            ****The essence of the project:**** 
+            *Describe the plot of the story in English, and the System will search the database for three films with a 
+            similar script.* \n\n
+            The system uses multi-agent architecture with GPU acceleration for optimal performance.
+            
+            **🤖 Powered by:** Nebius AI Studio | Modal Labs | FAISS | LlamaIndex ReAct Agents
+        """)
+
+        with gr.Row():
+            with gr.Column(scale=4):
+                # Добавляем type='messages'
+                chatbot = gr.Chatbot(
+                    value=[],
+                    height=600,
+                    label="🎬 Conversation with AI Agents (Local UI → Modal Agents → Nebius LLM)",
+                    show_copy_button=True,
+                    type='messages'  # Новый формат сообщений
+                )
+
+                msg = gr.Textbox(
+                    placeholder="Describe a movie plot (50-100 words in English)...",
+                    label="Your message",
+                    lines=3,
+                    max_lines=5
+                )
+
+                with gr.Row():
+                    submit_btn = gr.Button("🚀 Submit", variant="primary", scale=2)
+                    clear_btn = gr.Button("🔄 Clear Chat", scale=1)
+
+            with gr.Column(scale=1):
+                gr.Markdown("""
+                 ### 🔍 How to use:
+                
+                1. **📝 Describe the plot** (50-100 words in English)
+                2. **✅ Agent Editor validates** (length) and improves your description (grammar)
+                3. **🎬 Agent Film Critic** creates a movie overview based on your story
+                4. **🔍 System searches** the database for 10 films that correlate with your description 
+                5. **🎯 Agent Film Expert selects** top 3 movies with explanations
+                
+                ### 📋 Requirements:
+                - ✅ English text only
+                - ✅ 50-100 words
+                - ✅ Clear plot description
+                - ✅ Proper grammar (AI will help)
+                
+                ### ⚡ Features:
+                - 🚀 FAISS search
+                - 🧠 Multi-agent reasoning
+                - 📊 Semantic + narrative similarity
+                - 🎯 Expert film analysis
+                """)
+
+                session_info = gr.Textbox(
+                    label="Session Info",
+                    value=get_session_info(),
+                    interactive=False,
+                    lines=5
+                )
+
+                refresh_btn = gr.Button("🔄 Refresh Info", size="sm")
+
+        # Обработчики событий
+        submit_btn.click(
+            fn=chat_interface,
+            inputs=[msg, chatbot],
+            outputs=[chatbot, msg]
+        ).then(  # ✅ ДОБАВЛЕНО: Автообновление после отправки
+            fn=get_session_info,
+            outputs=[session_info]
+        )
+
+        msg.submit(
+            fn=chat_interface,
+            inputs=[msg, chatbot],
+            outputs=[chatbot, msg]
+        ).then(  # ✅ ДОБАВЛЕНО: Автообновление после Enter
+            fn=get_session_info,
+            outputs=[session_info]
+        )
+
+        clear_btn.click(
+            fn=reset_chat,
+            outputs=[chatbot, msg]
+        )
+
+        refresh_btn.click(
+            fn=force_refresh_session_info,
+            outputs=[session_info]
+        )
+
+# Запуск приложения
+    logger.info("Starting Movie Plot Search application")
+    demo.launch(
+        server_name="127.0.0.1",
+        server_port=7860,
+        share=True,
+        debug=False,
+        show_error=True
+    )
+
+
+# --- Функция запуска через Modal (если нужно полностью на Modal)---
+@app.function(secrets=[modal.Secret.from_name("nebius-secret")])
+def run_app():
+    _run_main_app()
+
+
+# --- Локальный запуск ---
+if __name__ == "__main__":
+    _run_main_app()

evaluation/__init__.py

@@ -0,0 +1,3 @@
+"""
+Evaluation module for shadow testing and LLM-as-a-Judge quality assessment.
+"""

evaluation/judges.py

@@ -0,0 +1,126 @@
+# evaluation/judges.py
+from agents.nebius_simple import create_nebius_llm
+import json
+import re
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class PersuasionJudge:
+    def __init__(self, api_key):
+        self.llm = create_nebius_llm(api_key, model="meta-llama/Llama-3.3-70B-Instruct", temperature=0.0)
+
+    def evaluate_expert_skill(self, user_story: str, expert_card: str, bridges: list) -> dict:
+        """
+        Оценивает мастерство Агента-Эксперта в написании рекомендаций.
+        """
+        bridges_str = "\n- ".join(bridges)
+
+        prompt = f"""
+        Act as a Senior Editor for a Movie Magazine.
+        Evaluate the quality and persuasiveness of the AI Critic's recommendation.
+
+        CONTEXT:
+        The User provided a detailed story (50+ words).
+        The AI Agent recommended a movie and wrote a "Justification".
+
+        --- USER STORY ---
+        "{user_story}"
+
+        --- AI RECOMMENDATION CARD ---
+        {expert_card}
+
+        --- EVALUATION TASK ---
+        Check if the AI Agent successfully built "Narrative Bridges" - connecting specific details from the User's story to the Movie.
+
+        Expected Bridges (The agent SHOULD mention these connections):
+        - {bridges_str}
+
+        Rate the Agent on 3 metrics (1-5 stars):
+
+        1. **Context Awareness (1-5)**: Did the agent reference specific details from the user's text (e.g. "You mentioned a botanist...") or did it use a generic template?
+        2. **Persuasiveness (1-5)**: Is the argument convincing? Does it explain WHY this movie matches the user's specific plot?
+        3. **Bridge Coverage (0-100%)**: What percentage of the "Expected Bridges" were explicitly addressed?
+
+        OUTPUT JSON ONLY:
+        {{
+            "context_score": int,
+            "persuasiveness_score": int,
+            "bridge_coverage_percent": int,
+            "missing_bridges": ["list of missed points"],
+            "feedback": "Short critique for the agent"
+        }}
+        """
+
+        try:
+            response = self.llm.complete(prompt).text
+            cleaned = re.sub(r"```json|```", "", response).strip()
+            return json.loads(cleaned)
+        except Exception as e:
+            return {"error": str(e), "persuasiveness_score": 0}
+
+    def evaluate_real_world_interaction(self, user_story: str, expert_card: str, movie_metadata: dict) -> dict:
+        """
+        Оценка реального диалога (Reference-Free).
+        Проверяет обоснованность (Groundedness) и логичность, не зная "правильного" ответа.
+        Разделяем "галлюцинации фактов" и "слабые тематические связи"
+        """
+        # Превращаем метаданные в текст для промпта
+        facts_str = json.dumps({
+            "title": movie_metadata.get("title"),
+            "director": movie_metadata.get("director"),
+            "cast": movie_metadata.get("cast"),
+            "genres": movie_metadata.get("genres"),
+            "plot_keywords": movie_metadata.get("narrative_features", "")
+        }, ensure_ascii=False)
+
+        prompt = f"""
+        You are an AI Auditor monitoring a Movie Recommendation System in production.
+        Your goal is to detect **Factual Hallucinations** and evaluate **Logical Coherence**.
+
+        --- INPUT DATA ---
+        1. USER STORY: "{user_story}"
+        2. REAL MOVIE FACTS (Ground Truth): {facts_str}
+        3. AGENT'S RECOMMENDATION TEXT: "{expert_card}"
+
+        --- AUDIT TASKS ---
+        **IMPORTANT DISTINCTION:**
+        - **Hallucination = Inventing facts that contradict Movie Facts** (e.g., wrong actors, wrong plot events)
+        - **Weak connection ≠ Hallucination** (e.g., "both are comedies with quirky characters" is NOT a hallucination, 
+        just a thematic bridge)
+        1. **Check Groundedness (Faithfulness)**: 
+              - Did the Agent mention any actors, directors, or plot details that CONTRADICT the Movie Facts?
+              - If the Agent describes plot events NOT in the movie's overview, that is a HALLUCINATION.
+              - If the Agent says "both films share a genre/mood/theme", that is NOT a hallucination.
+              - Score 0 (False claims) to 1 (Fully supported by facts).
+
+        2. **Check Logical Link**: 
+            - Does the Agent clearly explain *how* the movie connects to the User Story?
+            - Thematic connections ("both explore loneliness", "both are comedies") are VALID bridges.
+            - Score 1 (Vague) to 5 (Strong logic).
+
+        3. **Hallucination Detection**:
+           - Set `hallucination_detected: true` ONLY if the Agent invented false factual claims.
+           - Examples of hallucinations: wrong actors, fabricated plot events, fake quotes.
+           - Examples of NOT hallucinations: "both films share comedic tone", "similar narrative structure", 
+           "focuses on same themes".
+
+        OUTPUT JSON ONLY:
+        {{
+            "groundedness_score": float, 
+            "coherence_score": int,
+            "hallucination_detected": boolean,
+            "hallucination_details": "string (what was invented?) or null",
+            "reasoning": "Short audit report"
+        }}
+        """
+
+        try:
+            response = self.llm.complete(prompt).text
+            # Очистка JSON
+            cleaned = re.sub(r"```json|```", "", response).strip()
+            return json.loads(cleaned)
+        except Exception as e:
+            logger.error(f"Shadow eval failed: {e}")
+            return {"error": str(e), "groundedness_score": 0}

evaluation/run_evals.py

@@ -0,0 +1,101 @@
+# evaluation/run_evals.py
+import asyncio
+import json
+import os
+import datetime
+from agents.retriever import RetrieverAgent
+from agents.modal_agents import process_expert_agent
+from evaluation.judges import PersuasionJudge
+
+
+async def run_persuasion_eval():
+    nebius_key = os.environ.get("NEBIUS_API_KEY")
+    if not nebius_key:
+        print("❌ Error: NEBIUS_API_KEY environment variable is not set.")
+        return
+
+    # Инициализация
+    judge = PersuasionJudge(nebius_key)
+    retriever = RetrieverAgent()
+
+    # Загрузка датасета
+    try:
+        with open("evaluation/golden_dataset.json", "r") as f:
+            dataset = json.load(f)
+    except FileNotFoundError:
+        print("❌ Error: evaluation/golden_dataset.json not found.")
+        return
+
+    print(f"🕵️ Starting Evaluation on {len(dataset)} scenarios...")
+
+    report = {
+        "timestamp": datetime.datetime.utcnow().isoformat(),
+        "total_cases": len(dataset),
+        "results": [],
+        "summary": {}
+    }
+
+    total_persuasiveness = 0
+    total_context_score = 0
+
+    for case in dataset:
+        print(f"\nProcessing CASE ID: {case['id']}...")
+
+        # 1. Поиск
+        retrieval = retriever.retrieve_candidates(case['query'], top_k=10)
+        candidates = retrieval.get("candidates", [])
+
+        # 2. Эксперт
+        if candidates:
+            # Используем .remote для синхронного вызова (или aio для асинхронного, если настроено)
+            # В локальном скрипте проще использовать синхронный вызов к remote функции
+            expert_result = process_expert_agent.remote(case['query'], candidates)
+            expert_text = str(expert_result)
+            if isinstance(expert_result, dict):
+                expert_text = expert_result.get("explanations", str(expert_result))
+        else:
+            expert_text = ""
+
+        # 3. Судья
+        verdict = judge.evaluate_expert_skill(
+            user_story=case['query'],
+            expert_card=expert_text,
+            bridges=case.get('key_narrative_bridges', [])
+        )
+
+        # Сбор метрик
+        p_score = verdict.get('persuasiveness_score', 0)
+        c_score = verdict.get('context_score', 0)
+        total_persuasiveness += p_score
+        total_context_score += c_score
+
+        print(f"   Score: {p_score}/5 | Context: {c_score}/5")
+
+        # Добавляем в отчет
+        report["results"].append({
+            "case_id": case["id"],
+            "query": case["query"],
+            "expected_movie": case.get("expected_movie"),
+            "expert_output_snippet": expert_text[:200] + "...",
+            "scores": verdict,
+            "candidates_found": len(candidates)
+        })
+
+    # Итоговая статистика
+    report["summary"] = {
+        "avg_persuasiveness": round(total_persuasiveness / len(dataset), 2),
+        "avg_context_aware": round(total_context_score / len(dataset), 2)
+    }
+
+    # Сохранение в файл
+    filename = f"evaluation/report_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+    with open(filename, "w") as f:
+        json.dump(report, f, indent=2, ensure_ascii=False)
+
+    print(f"\n✅ Evaluation Complete!")
+    print(f"🏆 Average Persuasiveness: {report['summary']['avg_persuasiveness']}/5")
+    print(f"📄 Full report saved to: {filename}")
+
+
+if __name__ == "__main__":
+    asyncio.run(run_persuasion_eval())

memory/session_store.py

@@ -0,0 +1,89 @@
+# memory/session_store.py
+import uuid
+import datetime
+import logging
+from typing import Dict, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SessionStore:
+    """
+    Хранилище состояния сессии.
+    Полностью поддерживает логику: 1/0/2 меню, Custom Plot Mode и счетчики попыток.
+    """
+
+    def __init__(self):
+        self._sessions: Dict[str, Dict[str, Any]] = {}
+
+    def create_session(self) -> str:
+        """Создает новую сессию с полным набором полей"""
+        session_id = str(uuid.uuid4())
+        self._sessions[session_id] = {
+            "session_id": session_id,
+            "created_at": datetime.datetime.utcnow().isoformat(),
+
+            # --- Логика машины состояний ---
+            "step": "initial",
+            "attempts": 0,
+            "suggested_plot": None,  # Хранит текст предложенной истории
+            "custom_plot_mode": False,  # ✅ Флаг строгого режима (критично для вашей логики)
+
+            # --- Данные диалога ---
+            "history": [],
+            "user_inputs": [],
+
+            # --- Результаты текущего поиска ---
+            "original_plot": "",
+            "improved_plot": "",
+            "movie_overview": "",
+            "search_results": [],
+            "final_recommendations": []
+        }
+        return session_id
+
+    def get_session(self, session_id: str) -> Optional[Dict[str, Any]]:
+        """Возвращает копию состояния сессии"""
+        return self._sessions.get(session_id)
+
+    # --- Методы управления состоянием ---
+
+    def add_user_input(self, session_id: str, text: str):
+        if session_id in self._sessions:
+            self._sessions[session_id]["user_inputs"].append(text)
+
+    def add_history(self, session_id: str, user_msg: str, assistant_msg: str):
+        if session_id in self._sessions:
+            self._sessions[session_id]["history"].append({"role": "user", "content": user_msg})
+            self._sessions[session_id]["history"].append({"role": "assistant", "content": assistant_msg})
+
+    def update_state(self, session_id: str, key: str, value: Any):
+        """Универсальное обновление поля"""
+        if session_id in self._sessions:
+            self._sessions[session_id][key] = value
+
+    def increment_attempts(self, session_id: str):
+        if session_id in self._sessions:
+            self._sessions[session_id]["attempts"] += 1
+
+    def reset_attempts(self, session_id: str):
+        if session_id in self._sessions:
+            self._sessions[session_id]["attempts"] = 0
+
+    def set_custom_mode(self, session_id: str, is_active: bool):
+        """Управление флагом строгого режима"""
+        if session_id in self._sessions:
+            self._sessions[session_id]["custom_plot_mode"] = is_active
+
+    def clear_session_data(self, session_id: str):
+        """Мягкий сброс (для кнопки Reset)"""
+        if session_id in self._sessions:
+            base = self._sessions[session_id]
+            # Сбрасываем динамические данные
+            base["history"] = []
+            base["user_inputs"] = []
+            base["original_plot"] = ""
+            base["final_recommendations"] = []
+            base["attempts"] = 0
+            base["suggested_plot"] = None
+            base["custom_plot_mode"] = False

modal_app.py

@@ -0,0 +1,1075 @@
+import modal
+from modal import Image, App, Volume, Secret
+import logging
+# import modal
+import faiss
+import numpy as np
+import pandas as pd
+import pickle
+from sentence_transformers import SentenceTransformer
+import torch
+import os
+
+from agents.modal_agents import app as agents_app
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+image = (
+    Image.from_registry(
+        "nvidia/cuda:12.8.1-devel-ubuntu22.04",
+        add_python="3.11"
+    )
+    .apt_install(
+        "build-essential",
+        "python3-dev",
+        "gcc",
+        "g++",
+        "cmake",
+        "wget",  # Добавляем wget
+        "unzip"  # Добавляем unzip для распаковки
+    )
+    .pip_install_from_requirements("requirements_modal.txt")
+    .add_local_file(
+        local_path="setup_image.py",
+        remote_path="/root/setup_image.py",
+        copy=True
+    )
+    # ✅ ДОБАВЛЕНО: Добавляем скрипт извлечения punkt данных
+    .add_local_file(
+        local_path="setup_punkt_extraction.py",
+        remote_path="/root/setup_punkt_extraction.py",
+        copy=True
+    )
+
+    .run_commands("python /root/setup_image.py")
+    .run_commands("python -m spacy download en_core_web_lg")
+
+
+    .run_commands(
+        # Скачиваем ресурс punkt
+        "wget -O /tmp/punkt.zip https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip",
+        "unzip /tmp/punkt.zip -d /tmp",
+
+        # Создаем структуру директорий
+        "mkdir -p /root/nltk_data/tokenizers/punkt_tab/english",
+
+        # Копируем основные файлы
+        "cp /tmp/punkt/PY3/english.pickle /root/nltk_data/tokenizers/punkt_tab/english/",
+        "cp /tmp/punkt/README /root/nltk_data/tokenizers/punkt_tab/",
+        "cp -r /tmp/punkt/PY3 /root/nltk_data/tokenizers/punkt_tab/",
+
+        # ✅ КЛЮЧЕВОЕ ИЗМЕНЕНИЕ: Запускаем скрипт извлечения данных
+        "python /root/setup_punkt_extraction.py",
+
+        # Удаляем временные файлы
+        "rm -rf /tmp/punkt*"
+    )
+    .add_local_dir("modal_utils", remote_path="/root/modal_utils")
+    # .add_local_dir("local_utils", remote_path="/root/local_utils")
+    .add_local_dir("agents", remote_path="/root/agents")
+    .add_local_dir("evaluation", remote_path="/root/evaluation")
+)
+
+app = App(
+    name="tmdb-project",
+    image=image,
+    secrets=[
+        Secret.from_name("my-env"),  # Для конфиденциальных данных
+        Secret.from_name("nebius-secret")
+    ]
+)
+
+# Включаем все функции агентов в основной app
+app.include(agents_app)
+
+volume = Volume.from_name("tmdb-data", create_if_missing=True)
+
+# ✅ ДОБАВЛЯЕМ СЛОВАРЬ ДЛЯ СЕМАФОРА
+# Он будет хранить активные сессии: {session_id: timestamp}
+active_users_dict = modal.Dict.from_name("cinematch-active-users", create_if_missing=True)
+
+
+@app.function(
+    volumes={"/data": volume},
+    gpu="A10G",
+    timeout=3600
+)
+def process_movies():
+    """Основная функция обработки фильмов"""
+    # Импорт внутри функции для работы с добавленными директориями
+    from modal_utils.cloud_operations import heavy_computation
+    return heavy_computation()
+
+
+@app.function(volumes={"/data": volume})
+def upload_file(data_str: str):
+    import shutil
+    import os
+
+    volume.listdir(path='/', recursive=True)
+    print(volume.listdir(path='/', recursive=True))
+
+    # local_file_path = 'temp_sample.csv'  # Используйте временный файл
+    remote_file_path = '/data/input.csv'  # Путь в Volume
+
+    print(1)
+    # Создаем директорию, если нужно
+    os.makedirs(os.path.dirname(remote_file_path), exist_ok=True)
+
+    # Записываем данные напрямую в файл
+    with open(remote_file_path, "w") as f:
+        f.write(data_str)
+
+    print(f"Данные успешно записаны в Volume: {remote_file_path}")
+    return remote_file_path
+
+
+@app.function(
+    image=image,
+    gpu="A10G",  # было any
+    volumes={"/data": volume},
+    timeout=120  # было 1800 == 30 минут на батч
+)
+def process_batch(batch: list[tuple]):
+    """
+    Обрабатывает батч данных на GPU
+    Вход: список кортежей (processed_text, original_text)
+    Выход: список JSON-строк с признаками
+    """
+    import spacy
+    from textacy.extract import keyterms
+    from textblob import TextBlob
+    import numpy as np
+    import json
+    import en_core_web_lg  # Прямой импорт модели
+    import torch
+    from concurrent.futures import ThreadPoolExecutor
+
+    torch.set_num_threads(1)  # Уменьшаем число CPU потоков
+    spacy.prefer_gpu()  # Активирует GPU для spaCy
+
+    # Загружаем модель
+    nlp = en_core_web_lg.load()
+
+    # Добавляем sentencizer, если его нет в пайплайне
+    if "sentencizer" not in nlp.pipe_names:
+        nlp.add_pipe("sentencizer")
+
+    processed_texts = [item[0] for item in batch]
+    original_texts = [item[1] for item in batch]
+
+    # Обработка предобработанных текстов (Оптимизированная обработка spaCy)
+    # processed_docs = list(nlp.pipe(processed_texts, batch_size=128))
+    processed_docs = list(nlp.pipe(processed_texts, batch_size=4096))  # Увеличьте для GPU - было 400 для CPU
+
+    # Функция для параллельного вычисления эмоциональной вариативности (sentiment variance)
+    def compute_sentiment_variance(text):
+        if not text or len(text) < 20:
+            return 0.0
+
+        try:
+            blob = TextBlob(text)
+            if len(blob.sentences) > 1:
+                sentiments = [s.sentiment.polarity for s in blob.sentences]
+                return float(np.var(sentiments))
+            return 0.0
+        except:
+            return 0.0
+
+    # Параллельное вычисление для всего батча
+    with ThreadPoolExecutor(max_workers=16) as executor:
+        sentiment_variances = list(executor.map(compute_sentiment_variance, original_texts))
+
+    # Предварительно вычисляем plot_turns для всего батча
+    turn_keywords = {"but", "however", "though", "although", "nevertheless",
+                     "suddenly", "unexpectedly", "surprisingly", "abruptly"}
+
+    # Векторизованный расчет plot_turns.
+    # Используем более эффективный метод
+    lower_texts = [text.lower() for text in original_texts]
+    plot_turns_counts = [
+        sum(text.count(kw) for kw in turn_keywords)
+        if text and len(text) >= 20 else 0
+        for text in lower_texts
+    ]
+
+    results = []
+    for i, (processed_doc, original_text) in enumerate(zip(processed_docs, original_texts)):
+        features = {
+            "conflict_keywords": [],
+            "plot_turns": plot_turns_counts[i],  # Используем предвычисленное значение (было 0, вычислялось позже)
+            "sentiment_variance": sentiment_variances[i],  # Используем предвычисленное значение (было 0.0)
+            "action_density": 0.0
+        }
+
+        try:
+            # 1. Ключевые слова конфликта
+            if processed_texts[i] and len(processed_texts[i]) >= 20:
+                conflict_terms = [
+                    term for term, score in keyterms.textrank(
+                        processed_doc,
+                        topn=5,
+                        window_size=10,
+                        edge_weighting="count",
+                        position_bias=False
+                    ) if term and term.strip()
+                ]
+                features["conflict_keywords"] = conflict_terms
+
+            # Плотность действий
+            if processed_doc and len(processed_doc) > 0:
+                action_verbs = sum(1 for token in processed_doc if token.pos_ == "VERB")
+                features["action_density"] = action_verbs / len(processed_doc)
+
+        except Exception as e:
+            print(f"Error processing item {i}: {str(e)[:100]}")
+
+        results.append(json.dumps(features))
+
+    return results
+
+
+@app.function(
+    image=image,
+    volumes={"/data": volume},
+    # memory=6144,  # Увеличиваем память до 6 ГБ
+    timeout=600  # 150 минут вместо 60 секунд
+)
+def load_data(max_rows: int = None):
+    """Загружает данные из CSV на Volume"""
+    import pandas as pd
+
+    # file_path = "/data/data/output.csv"
+    file_path = "/data/data/output.parquet"  # Теперь используем Parquet
+    print(f"Loading data from {file_path}...")
+
+    # Чтение данных с возможностью ограничения количества строк
+    if max_rows:
+        # df = pd.read_csv(file_path, nrows=max_rows)
+        df = pd.read_parquet(file_path, rows=max_rows)
+    else:
+        # Чтение всего файла
+        df = pd.read_parquet(file_path)
+        # df = pd.read_csv(file_path)
+
+    print(f"Loaded {len(df)} records")
+
+    # Проверка необходимых столбцов
+    required_columns = ['processed_overview', 'overview']
+    for col in required_columns:
+        if col not in df.columns:
+            raise ValueError(f"Column '{col}' not found in dataset")
+    print(f'Columns check finished')
+
+    # Заполнение пропущенных значений
+    df['processed_overview'] = df['processed_overview'].fillna('')
+    df['overview'] = df['overview'].fillna('')
+    print(f'Missing values filling is finished')
+
+    return df
+
+
+@app.function(
+    image=image,
+    volumes={"/data": volume},
+    timeout=300  # 5 минут вместо 60 секунд
+)
+def save_results(df, output_path):
+    """Сохраняет результаты на Volume"""
+    print(f"Saving results to {output_path}...")
+    # df.to_parquet(output_path, index=False)
+    df.to_parquet(output_path, index=False, engine='pyarrow')  # или engine='fastparquet'
+    print(f"✅ Results saved to {output_path}")
+
+
+@app.local_entrypoint()
+def process_test_batch(batch_size: int = 1000):
+    """Обрабатывает тестовый батч из Volume"""
+    import json
+    # Загрузка данных
+    df = load_data.remote(max_rows=batch_size)
+
+    # Формирование батча
+    batch_data = list(zip(
+        df['processed_overview'].astype(str),
+        df['overview'].astype(str)
+    ))
+
+    # Обработка батча
+    print(f"Processing test batch ({len(batch_data)} records) on GPU...")
+    results = process_batch.remote(batch_data)
+
+    # Добавление результатов
+    df['narrative_features'] = results
+    df['features_decoded'] = df['narrative_features'].apply(json.loads)
+
+    # Сохранение результатов
+    output_path = f"/data/data/test_batch_results_{batch_size}.parquet"
+    save_results.remote(df, output_path)
+
+    # Вывод статистики
+    print("\nProcessing statistics:")
+    print(
+        f"Conflict_keywords (non-empty): {sum(1 for x in df['features_decoded'] if x['conflict_keywords'])}/{len(df)}")
+    print(f"Avg plot_turns: {df['features_decoded'].apply(lambda x: x['plot_turns']).mean():.2f}")
+    print(f"Avg sentiment_variance: {df['features_decoded'].apply(lambda x: x['sentiment_variance']).mean():.4f}")
+    print(f"Avg action_density: {df['features_decoded'].apply(lambda x: x['action_density']).mean():.2f}")
+
+    print("\n✅ Test batch processing complete!")
+
+
+@app.local_entrypoint()
+def show_sample_results(file_path: str = "/data/test_batch_results_1000.parquet"):
+    """Показывает примеры результатов из файла на Volume"""
+    import json
+
+    # Загрузка результатов
+    @app.function(volumes={"/data": volume})
+    def load_results(path):
+        import pandas as pd
+
+        return pd.read_parquet(path)
+
+    df = load_results.remote(file_path)
+
+    # Добавление декодированных признаков
+    if 'narrative_features' in df.columns:
+        df['features_decoded'] = df['narrative_features'].apply(json.loads)
+
+    print(f"Results from {file_path} ({len(df)} records):")
+
+    # Вывод примеров
+    sample_size = min(3, len(df))
+    print(f"\nSample of {sample_size} records:")
+    for i, row in df.head(sample_size).iterrows():
+        print(f"\nRecord {i}:")
+        print(f"Processed: {row['processed_overview'][:100]}...")
+        print(f"Original: {row['overview'][:100]}...")
+        print("Features:")
+        features = row['features_decoded'] if 'features_decoded' in row else json.loads(row['narrative_features'])
+        for k, v in features.items():
+            print(f"  {k}: {v}")
+
+    # Общая статистика
+    if 'features_decoded' in df.columns:
+        print("\nDataset statistics:")
+        print(f"Avg plot_turns: {df['features_decoded'].apply(lambda x: x['plot_turns']).mean():.2f}")
+        print(f"Avg sentiment_variance: {df['features_decoded'].apply(lambda x: x['sentiment_variance']).mean():.4f}")
+        print(f"Avg action_density: {df['features_decoded'].apply(lambda x: x['action_density']).mean():.2f}")
+
+
+@app.function(
+    image=image,
+    volumes={"/data": volume},
+    timeout=3600,  # 1 час на конвертацию
+    memory=8192  # 8 ГБ памяти
+)
+def convert_csv_to_parquet():
+    import pandas as pd
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+    import pyarrow.csv as pc
+    from pathlib import Path
+    import time
+
+    start_time = time.time()
+
+    input_path = "/data/data/output.csv"
+    output_path = "/data/data/output.parquet"
+
+    print(f"Starting conversion: {input_path} -> {output_path}")
+
+    # Создаем директорию если нужно
+    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+
+    # Читаем CSV с помощью PyArrow (оптимизировано для больших файлов)
+    reader = pc.open_csv(
+        input_path,
+        read_options=pc.ReadOptions(block_size=128 * 1024 * 1024),  # 128MB блоки
+        parse_options=pc.ParseOptions(delimiter=",")
+    )
+
+    # Схема для записи Parquet
+    writer = None
+
+    # Обрабатываем данные порциями
+    batch_count = 0
+    while True:
+        try:
+            batch = reader.read_next_batch()
+            if not batch:
+                break
+
+            df = batch.to_pandas()
+
+            if writer is None:
+                # Инициализируем writer при первом батче
+                writer = pq.ParquetWriter(
+                    output_path,
+                    pa.Table.from_pandas(df).schema,
+                    compression='SNAPPY'
+                )
+
+            # Конвертируем в pyarrow Table и записываем
+            table = pa.Table.from_pandas(df)
+            writer.write_table(table)
+
+            batch_count += 1
+            print(f"Processed batch {batch_count} ({df.shape[0]} rows)")
+
+        except StopIteration:
+            break
+
+    # Финализируем запись
+    if writer:
+        writer.close()
+
+    duration = time.time() - start_time
+    print(f"✅ Conversion complete! Saved to {output_path}")
+    print(f"Total batches: {batch_count}")
+    print(f"Total time: {duration:.2f} seconds")
+
+    return output_path
+
+
+@app.function(
+    image=image,
+    volumes={"/data": volume},
+    timeout=3600
+)
+def rebuild_parquet_with_row_index():
+    import pandas as pd
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    input_path = "/data/data/output.parquet"
+    output_path = "/data/data/output_indexed.parquet"
+
+    # Читаем исходные данные
+    df = pd.read_parquet(input_path)
+
+    # Добавляем индекс строки
+    df.reset_index(inplace=True)
+    df.rename(columns={'index': 'row_id'}, inplace=True)
+
+    # Сохраняем с разбивкой по строкам
+    table = pa.Table.from_pandas(df)
+    pq.write_table(table, output_path, row_group_size=15000)
+
+    return output_path
+
+
+@app.local_entrypoint()
+def process_full_dataset(batch_size: int = 15000):
+    """Обрабатывает и сохраняет результаты напрямую в Volume"""
+    from tqdm import tqdm
+    import math
+
+    # 1. Получаем только метаданные (количество строк)
+    total_records = get_row_count.remote()
+    print(f"Total records to process: {total_records}")
+
+    # 2. Рассчитываем количество батчей
+    num_batches = math.ceil(total_records / batch_size)
+    print(f"Processing in {num_batches} batches of {batch_size} records")
+
+    # 3. Создаем временную директорию для частичных результатов
+    partial_dir = "/data/partial_results"
+
+    # 4. Объединяем результаты
+    final_path = "/data/data/full_dataset_results.parquet"
+    combine_results.remote(partial_dir, final_path)
+
+    print("\n✅ Full dataset processing complete!")
+    print(f"Results saved to {final_path}")
+
+
+@app.function(volumes={"/data": volume})
+def init_partial_dir(partial_dir: str):
+    """Создает директорию для частичных результатов"""
+    import os
+    os.makedirs(partial_dir, exist_ok=True)
+    return f"Created directory {partial_dir}"
+
+
+@app.function(
+    image=image,
+    gpu="A10G",
+    volumes={"/data": volume},
+    timeout=300
+)
+def process_and_save_batch(start_row: int, end_row: int, batch_idx: int, partial_dir: str):
+    """Обрабатывает батч и сохраняет результаты в отдельный файл"""
+    import pandas as pd
+    import pyarrow.parquet as pq
+    import os
+
+    # 0. Создаем директорию, если ее нет
+    os.makedirs(partial_dir, exist_ok=True)
+
+    # 1. Чтение данных
+    file_path = "/data/data/output.parquet"
+
+    # Альтернативный метод чтения без row groups
+    df = pd.read_parquet(file_path)
+    df = df.iloc[start_row:end_row]
+
+    # 2. Подготовка данных
+    df['processed_overview'] = df['processed_overview'].fillna('')
+    df['overview'] = df['overview'].fillna('')
+
+    # 3. Формирование батча
+    batch_data = list(zip(
+        df['processed_overview'].astype(str),
+        df['overview'].astype(str)
+    ))
+
+    # 4. Обработка батча
+    results = process_batch.remote(batch_data)
+
+    # 5. Сохранение результатов
+    result_df = pd.DataFrame({'narrative_features': results})
+    output_path = os.path.join(partial_dir, f"batch_{batch_idx}.parquet")
+    result_df.to_parquet(output_path)
+
+    return f"Saved batch {batch_idx} to {output_path}"
+
+
+@app.function(volumes={"/data": volume})
+def combine_results(partial_dir: str, final_path: str):
+    """Объединяет частичные результаты в финальный файл"""
+    import pandas as pd
+    import os
+    from glob import glob
+    # import pyarrow.parquet as pq
+
+    # 1. Сбор всех частичных файлов
+    # partial_files = glob(os.path.join(partial_dir, "*.parquet"))
+    partial_files = sorted(
+        glob(os.path.join(partial_dir, "*.parquet")),
+        key=lambda x: int(os.path.basename(x).split('_')[1].split('.')[0])
+    )
+    print(partial_files)
+    # 2. Чтение и объединение
+    full_results = []
+    for file_path in partial_files:
+        df = pd.read_parquet(file_path)
+        full_results.extend(df['narrative_features'].tolist())
+
+    print(f'len(full_results) = {len(full_results)}')
+    # 3. Чтение исходных данных
+    source_df = pd.read_parquet("/data/data/output.parquet")
+    print({source_df.info()})
+
+    # 4. Добавляем результаты
+    source_df['narrative_features'] = full_results
+
+    # 5. Сохранение финального результата
+    source_df.to_parquet(final_path)
+
+    # 6. Очистка временных файлов
+    for file_path in partial_files:
+        os.remove(file_path)
+    os.rmdir(partial_dir)
+
+    return f"Combined {len(partial_files)} batches into {final_path}"
+
+
+@app.function(volumes={"/data": volume})
+def get_row_count():
+    """Возвращает общее количество строк в Parquet файле"""
+    import pyarrow.parquet as pq
+
+    file_path = "/data/data/output.parquet"
+    return pq.read_metadata(file_path).num_rows
+
+
+@app.function(
+    image=image,
+    volumes={"/data": volume},
+    gpu="A10G",
+    timeout=3600,
+    memory=16384
+)
+def build_faiss_index():
+    """
+    Построение FAISS индекса с учетом совместимости CUDA 12.8
+        Исправленная версия для эмбеддингов в формате строкового Python списка
+    """
+    import ast
+
+    print("Проверка доступности CUDA...")
+    print(f"CUDA доступна: {torch.cuda.is_available()}")
+    if torch.cuda.is_available():
+        print(f"CUDA устройств: {torch.cuda.device_count()}")
+        print(f"Текущее устройство: {torch.cuda.current_device()}")
+
+    # Загрузка данных
+    df = pd.read_parquet("/data/data/full_dataset_results.parquet")
+    print(f"Загружено {len(df)} фильмов")
+
+    # Анализ формата первого эмбеддинга
+    sample_embedding = df['processed_overview_embedding'].iloc[0]
+    print(f"Пример эмбеддинга: {str(sample_embedding)[:100]}...")
+    print(f"Тип данных: {type(sample_embedding)}")
+
+    # Извлечение эмбеддингов
+    embeddings_list = []
+    valid_indices = []
+    parse_errors = 0
+
+    print("Начинаем обработку эмбеддингов...")
+
+    for idx, (_, row) in enumerate(df.iterrows()):
+        try:
+            embedding_data = row['processed_overview_embedding']
+
+            # Обработка различных форматов хранения эмбеддингов.
+            # А именно - парсинг строкового представления Python списка
+            if isinstance(embedding_data, str):
+                try:
+                    # Безопасный парсинг с помощью ast.literal_eval
+                    parsed_list = ast.literal_eval(embedding_data.strip())
+                    if isinstance(parsed_list, list):
+                        embedding = np.array(parsed_list, dtype=np.float32)
+                    else:
+                        parse_errors += 1
+                        continue
+                except (ValueError, SyntaxError):
+                    parse_errors += 1
+                    continue
+            elif isinstance(embedding_data, list):
+                embedding = np.array(embedding_data, dtype=np.float32)
+            elif isinstance(embedding_data, np.ndarray):
+                embedding = embedding_data.astype(np.float32)
+            else:
+                parse_errors += 1
+                continue
+
+            # Проверка размерности (Размерность all-MiniLM-L6-v2 = 384)
+            if len(embedding) == 384:
+                embeddings_list.append(embedding.astype(np.float32))
+                valid_indices.append(idx)
+            else:
+                parse_errors += 1
+
+        except Exception as e:
+            parse_errors += 1
+            if parse_errors <= 5:  # Выводим первые несколько ошибок
+                print(f"Ошибка обработки эмбеддинга {idx}: {e}")
+            continue
+
+        # Прогресс каждые 50000 записей
+        if (idx + 1) % 50000 == 0:
+            print(f"Обработано {idx + 1}/{len(df)} записей, валидных: {len(embeddings_list)}")
+
+    print(f"Успешно обработано {len(embeddings_list)} эмбеддингов из {len(df)}")
+    print(f"Ошибок парсинга: {parse_errors}")
+    print(f"Успешность обработки: {len(embeddings_list) / len(df) * 100:.2f}%")
+
+    if not embeddings_list:
+        raise ValueError(f"Не найдено валидных эмбеддингов. Всего ошибок: {parse_errors}")
+
+    # Создание матрицы эмбеддингов
+    embeddings_matrix = np.vstack(embeddings_list)
+    print(f"Подготовлено {len(embeddings_matrix)} эмбеддингов")
+    print(f"Создана матрица эмбеддингов: {embeddings_matrix.shape}")
+
+    # Нормализация для косинусного сходства
+    faiss.normalize_L2(embeddings_matrix)
+    print("Эмбеддинги нормализованы")
+
+    # Создание FAISS индекса с поддержкой GPU
+    dimension = embeddings_matrix.shape[1]
+
+    # Проверяем доступность GPU для FAISS
+    if faiss.get_num_gpus() > 0:
+        print("Используем GPU для построения FAISS индекса")
+        # GPU ресурсы
+        res = faiss.StandardGpuResources()
+
+        # CPU индекс
+        cpu_index = faiss.IndexFlatIP(dimension)
+
+        # Перенос на GPU
+        gpu_index = faiss.index_cpu_to_gpu(res, 0, cpu_index)
+        gpu_index.add(embeddings_matrix)
+
+        # Возврат на CPU для сохранения
+        index = faiss.index_gpu_to_cpu(gpu_index)
+        print("FAISS индекс построен на GPU и перенесен на CPU для сохранения")
+    else:
+        print("Используем CPU для FAISS")
+        index = faiss.IndexFlatIP(dimension)
+        index.add(embeddings_matrix)
+
+    # Сохранение результатов
+    print("Сохранение FAISS индекса...")
+    faiss.write_index(index, "/data/data/movie_embeddings.index")
+
+    # Сохранение метаданных
+    print("Сохранение метаданных фильмов...")
+    valid_movies_df = df.iloc[valid_indices].reset_index(drop=True)
+    valid_movies_df.to_parquet("/data/data/indexed_movies_metadata.parquet")
+
+    result = {
+        "status": "success",
+        "total_movies": len(valid_movies_df),
+        "original_dataset_size": len(df),
+        "index_size": index.ntotal,
+        "dimension": dimension,
+        "gpu_used": faiss.get_num_gpus() > 0,
+        "processing_success_rate": len(valid_indices) / len(df),
+        "parse_errors": parse_errors
+    }
+
+    print("=" * 50)
+    print("ПОСТРОЕНИЕ ИНДЕКСА ЗАВЕРШЕНО")
+    print(f"Обработано фильмов: {result['total_movies']} из {result['original_dataset_size']}")
+    print(f"Размерность векторов: {result['dimension']}")
+    print(f"Успешность: {result['processing_success_rate'] * 100:.2f}%")
+    print("=" * 50)
+
+    return result
+
+
+@app.function(
+    image=image,
+    volumes={"/data": volume},
+    timeout=300
+)
+def test_embedding_parsing(num_samples=100):
+    """
+    Тестирование парсинга эмбеддингов на небольшой выборке данных
+    """
+    import ast
+
+    df = pd.read_parquet("/data/data/full_dataset_results.parquet")
+    print(f"Загружено {len(df)} фильмов для тестирования")
+
+    test_sample = df.head(num_samples)
+    successful_parses = 0
+    failed_parses = 0
+
+    print("Тестирование парсинга эмбеддингов...")
+
+    for idx, row in test_sample.iterrows():
+        embedding_data = row['processed_overview_embedding']
+
+        try:
+            if isinstance(embedding_data, str):
+                parsed_list = ast.literal_eval(embedding_data.strip())
+                if isinstance(parsed_list, list):
+                    embedding = np.array(parsed_list, dtype=np.float32)
+                    if len(embedding) == 384:
+                        successful_parses += 1
+                    else:
+                        print(f"Неправильная размерность {len(embedding)} для индекса {idx}")
+                        failed_parses += 1
+                else:
+                    print(f"Парсинг не дал список для индекса {idx}: {type(parsed_list)}")
+                    failed_parses += 1
+            else:
+                print(f"Неожиданный тип данных для индекса {idx}: {type(embedding_data)}")
+                failed_parses += 1
+
+        except Exception as e:
+            print(f"Ошибка парсинга для индекса {idx}: {e}")
+            failed_parses += 1
+
+    print(f"\nРезультаты тестирования:")
+    print(f"Успешных парсингов: {successful_parses}")
+    print(f"Неудачных парсингов: {failed_parses}")
+    print(f"Успешность: {successful_parses / (successful_parses + failed_parses) * 100:.2f}%")
+
+    return {
+        "successful_parses": successful_parses,
+        "failed_parses": failed_parses,
+        "success_rate": successful_parses / (successful_parses + failed_parses)
+    }
+
+
+@app.function(
+    image=image,
+    gpu="A10G",
+    timeout=300,
+    max_containers=1 # Макс 3 одновременных кодирования
+)
+def encode_user_query(query_text: str, remove_entities: bool = True):
+    """
+    Генерация эмбеддинга для пользовательского описания с опциональным удалением именованных сущностей
+    """
+    import spacy
+    import tempfile
+    # Импорт внутри функции для работы с добавленными директориями
+    from modal_utils.cloud_operations import (clean_text, prepare_text_for_embedding,
+                                              encode_user_query_fallback, extract_narrative_features_consistent)
+
+    # Проверка входных данных
+    if not query_text or not query_text.strip():
+        raise ValueError("Пустой запрос не может быть обработан")
+
+    # Определение устройства
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Используется устройство: {device}")
+
+    # Инициализация spaCy модели (та же, что использовалась для обработки фильмов)
+    try:
+        try:
+            # Загрузка spaCy с проверкой GPU
+            if torch.cuda.is_available():
+                spacy.prefer_gpu()
+
+            import en_core_web_lg
+            nlp = en_core_web_lg.load()
+            # nlp = spacy.load("en_core_web_lg")
+
+            # Добавляем sentencizer, если его нет
+            if "sentencizer" not in nlp.pipe_names:
+                nlp.add_pipe("sentencizer")
+
+            print("SpaCy модель загружена успешно")
+
+        except Exception as e:
+            print(f"Ошибка загрузки spaCy: {e}")
+            # Fallback к простой обработке
+            return encode_user_query_fallback(query_text, device)
+
+        # Инициализация модели для кодирования
+        try:
+            # Определяем кэш-директорию в зависимости от ОС
+            cache_dir = os.path.join(tempfile.gettempdir(), "sentence_transformer_cache")
+
+            # Создаем директорию, если не существует
+            os.makedirs(cache_dir, exist_ok=True)
+            print(f"Using cache directory: {cache_dir}")
+
+            model = SentenceTransformer(
+                'all-MiniLM-L6-v2',
+                device=device,
+                cache_folder=cache_dir
+            )
+
+            # Оптимизация для GPU
+            if torch.cuda.is_available():
+                model = model.half()
+                print("Using half-precision model")
+        except Exception as e:
+            return {"error": f"model_init_error: {str(e)}"}
+
+        # Применяем тот же процесс обработки, что и для фильмов
+        # Опциональное удаление именованных сущностей для фокуса на сюжете
+        if remove_entities:
+            processed_query = prepare_text_for_embedding(query_text, nlp)
+
+            # Проверка, что после обработки остался текст
+            if not processed_query.strip():
+                print("Предупреждение: После удаления сущностей текст стал пустым, используем очищенную версию")
+                processed_query = clean_text(query_text)
+        else:
+            processed_query = clean_text(query_text)
+
+        # Финальная проверка
+        if not processed_query.strip():
+            processed_query = query_text.lower().strip()
+
+        print(f"Исходное описание: '{query_text}'")
+        print(f"Обработанное описание: '{processed_query}'")
+
+        print(f"Исходное описание: {query_text}")
+        print(f"Обработанное описание: {processed_query}")
+
+        # Генерация эмбеддинга
+        query_embedding = model.encode(
+            [processed_query],
+            convert_to_tensor=False,
+            batch_size=1,
+            show_progress_bar=False
+        )[0]
+
+        # Извлечение нарративных признаков, консистентных с базой данных
+        narrative_features = extract_narrative_features_consistent(query_text, processed_query, nlp)
+
+        return {
+            "original_query": query_text,
+            "processed_query": processed_query,
+            "embedding": query_embedding.tolist(),
+            "embedding_dimension": len(query_embedding),
+            "narrative_features": narrative_features,
+            "device_used": device,
+            "preprocessing_applied": remove_entities
+        }
+    except Exception as e:
+        print(f"Ошибка в основной обработке: {e}, переключаемся на fallback")
+        return encode_user_query_fallback(query_text, device)
+
+
+@app.function(
+    image=image,
+    timeout=300
+)
+def test_text_processing_consistency():
+    """
+    Тестирование консистентности обработки текста между фильмами и описанием пошьзователя
+    Запуск из командной строки на локальном комп-ре:
+    $ modal run modal_app.py::app.test_text_processing_consistency
+    """
+    import spacy
+    # Импорт внутри функции для работы с добавленными директориями
+    from modal_utils.cloud_operations import clean_text, prepare_text_for_embedding, encode_user_query_fallback
+
+    nlp = spacy.load("en_core_web_lg")
+
+    # Тестовые примеры
+    test_texts = [
+        "A young wizard named Harry Potter discovers his magical heritage.",
+        "In New York City, a detective investigates a mysterious crime.",
+        "The story follows John Smith as he travels through time.",
+        "An epic adventure in the Star Wars universe with Luke Skywalker."
+    ]
+
+    print("Тестирование обработки текста:")
+    print("=" * 60)
+
+    for text in test_texts:
+        processed = prepare_text_for_embedding(text, nlp)
+        print(f"Исходный: {text}")
+        print(f"Обработанный: {processed}")
+        print("-" * 40)
+
+    return {"test_completed": True, "samples_processed": len(test_texts)}
+
+
+# Глобальная переменная для кэширования GPU индекса
+_gpu_index_cache = None
+_gpu_resources_cache = None
+
+
+@app.function(
+    image=image,
+    volumes={"/data": volume},
+    gpu="A10G",
+    timeout=300,
+    min_containers=1,  # Поддерживаем контейнер активным
+    max_containers=1 # Макс 3 одновременных кодирования
+)
+def search_similar_movies(
+        query_embedding: list,
+        query_narrative_features: dict,
+        top_k: int = 50,
+        rerank_top_n: int = 10):
+    """
+    Поиск похожих фильмов с использованием FAISS и консистентных нарративных признаков
+    дополнительным ранжированием
+    по нарративным признакам. Оптимизированная версия с кэшированием GPU
+    индекса для избежания повторных переносов
+    """
+    global _gpu_index_cache, _gpu_resources_cache
+    import time
+    from modal_utils.cloud_operations import (rerank_by_narrative_features,
+                                              calculate_narrative_similarity)
+
+    start_time = time.time()
+
+    search_index = None  # Инициализируем переменную
+
+    # Загрузка FAISS индекса
+    movies_df = pd.read_parquet("/data/data/indexed_movies_metadata.parquet")
+
+    # Инициализация GPU индекса (только при первом вызове)
+    if _gpu_index_cache is None and faiss.get_num_gpus() > 0:
+        print("Первая инициализация GPU индекса...")
+
+        # Загрузка CPU индекса
+        cpu_index = faiss.read_index("/data/data/movie_embeddings.index")
+
+        load_time = time.time() - start_time
+        print(f"Загрузка данных: {load_time:.3f}s")
+
+        # Создание GPU ресурсов
+        _gpu_resources_cache = faiss.StandardGpuResources()
+        _gpu_resources_cache.setTempMemory(1024 * 1024 * 1024)  # 1GB temp memory
+
+        # Перенос на GPU
+        _gpu_index_cache = faiss.index_cpu_to_gpu(_gpu_resources_cache, 0, cpu_index)
+
+        logger.info(f"GPU индекс кэширован и готов к использованию")
+        print("GPU индекс кэширован и готов к использованию")
+        using_gpu = True
+
+    elif _gpu_index_cache is not None:
+        logger.info(f"Используем кэшированный GPU индекс")
+        print("Используем кэшированный GPU индекс")
+        using_gpu = True
+
+    else:
+        logger.info(f"GPU недоступен, используем CPU")
+        print("GPU недоступен, используем CPU")
+        cpu_index = faiss.read_index("/data/data/movie_embeddings.index")
+        search_index = cpu_index
+        using_gpu = False
+
+    if using_gpu:
+        search_index = _gpu_index_cache
+
+    # Семантический поиск, Подготовка запроса
+    query_vector = np.array([query_embedding], dtype=np.float32)
+    faiss.normalize_L2(query_vector)
+
+    # Выполнение поиска ближайших соседей
+    search_start = time.time()
+    distances, indices = search_index.search(query_vector, top_k)
+    search_time = time.time() - search_start
+
+    logger.info(f"Время поиска ({'GPU' if using_gpu else 'CPU'}): {search_time:.3f}s")
+    print(f"Время поиска ({'GPU' if using_gpu else 'CPU'}): {search_time:.3f}s")
+
+    # Обработка результатов
+    process_start = time.time()
+    candidates = []
+    for i, (dist, idx) in enumerate(zip(distances[0], indices[0])):
+        if idx < len(movies_df):
+            movie = movies_df.iloc[idx]
+
+            # Вычисление нарративного сходства с исправленной функцией
+            narrative_similarity = calculate_narrative_similarity(
+                query_narrative_features,
+                movie.get('narrative_features', '{}')
+            )
+
+            candidates.append({
+                'index': idx,
+                'semantic_score': float(dist),
+                'narrative_similarity': narrative_similarity,
+                'movie_data': movie.to_dict()
+            })
+
+    # Дополнительное ранжирование с учетом нарративных признаков
+    reranked_candidates = rerank_by_narrative_features(candidates)
+    process_time = time.time() - process_start
+
+    total_time = time.time() - start_time
+
+    # Подготавливаем необходимые для инфо поля и выводим через logger
+    # filtered = {}
+    desired_movie_keys = {'id', 'title', 'narrative_features'}
+    if reranked_candidates:  # список не пуст
+        first = reranked_candidates[0]  # это dict
+        movie_info = first.get("movie_data", {})  # dict с данными фильма
+        filtered = {k: movie_info.get(k) for k in desired_movie_keys if k in movie_info}
+        logger.info(f"First re-ranked candidate (filtered): {filtered}")
+
+    else:
+        logger.warning("reranked_candidates is empty, nothing to log")
+
+    return {
+        "results": reranked_candidates[:rerank_top_n],
+        "performance_metrics": {
+            "using_gpu": using_gpu,
+            "search_time": search_time,
+            "process_time": process_time,
+            "total_time": total_time,
+            "cached_gpu_index": _gpu_index_cache is not None
+        }
+    }

modal_utils/cloud_operations.py

@@ -0,0 +1,429 @@
+import re
+import spacy
+
+import pandas as pd
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from tqdm import tqdm
+import torch
+import os
+import tempfile
+
+
+def clean_text(text):
+    """Очищает текст от лишних символов и форматирования"""
+    if pd.isna(text):
+        return ""
+
+    text = re.sub(r'[^a-zA-Z0-9\s]', ' ', text)  # Замена спецсимволов на пробелы
+    text = re.sub(r'\s+', ' ', text)       # Замена множественных пробелов на один
+    return text.strip().lower()
+
+
+def remove_entities(text, nlp):
+    """Удаляет именованные сущности, стоп-слова и приводит к леммам"""
+    if pd.isna(text) or text == "":
+        return ""
+
+    # nlp = spacy.load("en_core_web_lg")
+
+    doc = nlp(text)
+
+    # Удаляем именованные сущности, знаки пунктуации и стоп-слова,
+    # применяем лемматизацию к оставшимся токенам
+    tokens = [token.lemma_ for token in doc
+              if token.ent_type_ not in ['PERSON', 'ORG', 'GPE', 'LOC', 'DATE', 'TIME', 'MONEY']
+              and not token.is_punct
+              and not token.is_stop
+              and token.lemma_.strip()]
+
+    return " ".join(tokens)
+
+
+def prepare_text_for_embedding(text, nlp):
+    """Полный процесс подготовки текста для эмбеддинга"""
+    if pd.isna(text):
+        return ""
+
+    # Сначала очищаем текст
+    cleaned_text = clean_text(text)
+
+    # Затем удаляем сущности и выполняем лемматизацию
+    processed_text = remove_entities(cleaned_text, nlp)
+
+    return processed_text
+
+
+def encode_user_query_fallback(query_text: str, device: str):
+    """
+    Fallback функция для случаев, когда spaCy недоступна
+    """
+    import re
+
+    print("Fallback режим: обработка без spaCy")
+    # Инициализация модели SentenceTransformer
+    model = SentenceTransformer(
+        'all-MiniLM-L6-v2',
+        device=device,
+        cache_folder="/tmp/model_cache"
+    )
+
+    if torch.cuda.is_available():
+        model = model.half()
+
+    # Простая очистка без spaCy
+    processed_query = re.sub(r'[^a-zA-Z0-9\s]', ' ', query_text)
+    processed_query = re.sub(r'\s+', ' ', processed_query).strip().lower()
+
+    # Простое удаление потенциальных имен (заглавные буквы)
+    processed_query = re.sub(r'\b[A-Z][a-z]+\b', '', processed_query)
+    processed_query = re.sub(r'\s+', ' ', processed_query).strip()
+
+    print(f"Обработанный запрос: {processed_query}")
+
+    # Генерация эмбеддинга
+    query_embedding = model.encode(
+        [processed_query],
+        convert_to_tensor=False,
+        batch_size=1,
+        show_progress_bar=False
+    )[0]
+
+    # Базовые нарративные признаки без spaCy
+    narrative_features = {
+        "conflict_keywords": [],
+        "plot_turns": 0,
+        "sentiment_variance": 0.0,
+        "action_density": 0.0
+    }
+
+    return {
+        "original_query": query_text,
+        "processed_query": processed_query,
+        "embedding": query_embedding.tolist(),
+        "embedding_dimension": len(query_embedding),
+        "narrative_features": narrative_features,
+        "device_used": device,
+        "preprocessing_applied": True,
+        "fallback_mode": True
+    }
+
+
+def heavy_computation(df=None, batch_size=128):
+    """
+    Основная функция обработки, поддерживает два режима:
+    - Локальный: получает данные через параметр df
+    - Облачный: читает данные из Volume (/data/input.csv)
+    """
+    # Определяем режим выполнения
+    is_cloud_mode = df is None
+    print(f'is_cloud_mode = {is_cloud_mode}')
+    # 1. Загрузка данных (разные источники для локального и облачного режимов)
+    if is_cloud_mode:
+        # Облачный режим: читаем из Volume
+        try:
+            input_path = "/data/input.csv"
+            if not os.path.exists(input_path):
+                return {"error": "input_file_not_found"}
+
+            df = pd.read_csv(input_path)
+            print(f"Loaded {len(df)} movies from Volume")
+        except Exception as e:
+            return {"error": f"data_load_error: {str(e)}"}
+    else:
+        # Локальный режим: используем переданные данные
+        print(f"Processing {len(df)} movies locally")
+
+    # 2. Инициализация модели
+    try:
+        # Определяем кэш-директорию в зависимости от ОС
+        cache_dir = os.path.join(tempfile.gettempdir(), "sentence_transformers_cache")
+
+        # Создаем директорию, если не существует
+        os.makedirs(cache_dir, exist_ok=True)
+        print(f"Using cache directory: {cache_dir}")
+
+        model = SentenceTransformer(
+            'all-MiniLM-L6-v2',
+            device="cuda" if torch.cuda.is_available() else "cpu",
+            cache_folder=cache_dir
+        )
+
+        if torch.cuda.is_available():
+            model = model.half()
+            print("Using half-precision model")
+    except Exception as e:
+        return {"error": f"model_init_error: {str(e)}"}
+
+    # 3. Создание эмбеддингов
+    try:
+        embeddings = []
+        non_empty_overviews = df['processed_overview'].fillna("")
+
+        # Если нет GPU, уменьшаем размер батча
+        if not torch.cuda.is_available() and batch_size > 32:
+            batch_size = 32
+            print(f"Reduced batch_size to {batch_size} for CPU mode")
+
+        for i in tqdm(range(0, len(non_empty_overviews), batch_size),
+                      total=len(non_empty_overviews) // batch_size + 1):
+            batch = non_empty_overviews.iloc[i:i + batch_size].tolist()
+            batch_embeddings = model.encode(
+                batch,
+                show_progress_bar=False,
+                convert_to_numpy=True
+            )
+            embeddings.append(batch_embeddings.astype(np.float32))
+
+        # 4. Сохранение результатов
+        df['processed_overview_embedding'] = np.vstack(embeddings).tolist()
+        # df['title_length'] = df['title'].apply(len)
+        # df['has_overview'] = df['overview'].notna()
+
+        # Для локального режима просто возвращаем результат
+        if not is_cloud_mode:
+            return {
+                "status": "success",
+                "processed": len(df),
+                "embedding_dim": embeddings[0].shape[1] if embeddings else 0,
+                "sample": {
+                    "title": df['title'].iloc[0],
+                    "overview": df['overview'].iloc[0][:50] + "..." if len(df['overview'].iloc[0]) > 50 else df['overview'].iloc[0],
+                    "embedding_first_5": df['overview_embedding'].iloc[0][:5]
+    }
+            }
+
+        # 5. Для облачного режима сохраняем в Volume
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, encoding="utf-8") as tmp:
+            df.to_csv(tmp, index=False)
+            tmp_path = tmp.name
+
+        # Копируем в Volume
+        output_path = "/data/data/output.csv"
+        with open(tmp_path, "rb") as src, open(output_path, "wb") as dst:
+            dst.write(src.read())
+
+        os.unlink(tmp_path)
+
+        return {
+            "status": "success",
+            "processed": len(df),
+            "embedding_dim": embeddings[0].shape[1] if embeddings else 0,
+            "saved_path": output_path
+        }
+
+    except Exception as e:
+        return {
+            "status": "error",
+            "message": str(e),
+            "type": type(e).__name__
+        }
+
+
+def parse_embedding_safe(embedding_data):
+    """
+    Оптимизированная функция парсинга эмбеддингов для формата строкового Python списка
+    """
+    import ast
+
+    # Случай 1: Уже numpy array
+    if isinstance(embedding_data, np.ndarray):
+        return embedding_data
+
+    # Случай 2: Python список
+    if isinstance(embedding_data, list):
+        return np.array(embedding_data, dtype=np.float32)
+
+    # Случай 3: Строковое представление Python списка
+    if isinstance(embedding_data, str):
+        try:
+            # Используем ast.literal_eval для безопасного парсинга
+            parsed_list = ast.literal_eval(embedding_data.strip())
+            if isinstance(parsed_list, list):
+                return np.array(parsed_list, dtype=np.float32)
+        except (ValueError, SyntaxError) as e:
+            print(f"Ошибка парсинга эмбеддинга: {e}")
+            return None
+
+    return None
+
+
+def extract_narrative_features_consistent(original_text: str, processed_text: str, nlp):
+    """
+    Извлечение нарративных признаков, идентичных тем, что используются для фильмов
+    """
+    from textacy.extract import keyterms
+    from textblob import TextBlob
+    import numpy as np
+    import json
+
+    # Инициализация структуры признаков (как в базе данных)
+    features = {
+        "conflict_keywords": [],
+        "plot_turns": 0,
+        "sentiment_variance": 0.0,
+        "action_density": 0.0
+    }
+
+    try:
+        # Обработка текста через spaCy
+        processed_doc = nlp(processed_text) if processed_text and len(processed_text) >= 20 else None
+
+        # 1. Ключевые слова конфликта (идентичный алгоритм)
+        if processed_doc and len(processed_text) >= 20:
+            try:
+                conflict_terms = [
+                    term for term, score in keyterms.textrank(
+                        processed_doc,
+                        topn=5,
+                        window_size=10,
+                        edge_weighting="count",
+                        position_bias=False
+                    ) if term and term.strip()
+                ]
+                features["conflict_keywords"] = conflict_terms
+            except Exception as e:
+                print(f"Ошибка извлечения ключевых слов: {e}")
+                features["conflict_keywords"] = []
+
+        # 2. Повороты сюжета (идентичный алгоритм)
+        if original_text and len(original_text) >= 20:
+            turn_keywords = {"but", "however", "though", "although", "nevertheless",
+                             "suddenly", "unexpectedly", "surprisingly", "abruptly"}
+
+            lower_text = original_text.lower()
+            plot_turns_count = sum(lower_text.count(kw) for kw in turn_keywords)
+            features["plot_turns"] = plot_turns_count
+
+        # 3. Вариативность эмоций (идентичный алгоритм)
+        if original_text and len(original_text) >= 20:
+            try:
+                blob = TextBlob(original_text)
+                if len(blob.sentences) > 1:
+                    sentiments = [s.sentiment.polarity for s in blob.sentences]
+                    features["sentiment_variance"] = float(np.var(sentiments))
+                else:
+                    features["sentiment_variance"] = 0.0
+            except Exception as e:
+                print(f"Ошибка анализа эмоций: {e}")
+                features["sentiment_variance"] = 0.0
+
+        # 4. Плотность действий (идентичный алгоритм)
+        if processed_doc and len(processed_doc) > 0:
+            action_verbs = sum(1 for token in processed_doc if token.pos_ == "VERB")
+            features["action_density"] = action_verbs / len(processed_doc)
+
+    except Exception as e:
+        print(f"Ошибка извлечения нарративных признаков: {e}")
+
+    return features
+
+
+def calculate_narrative_similarity(query_features, movie_features):
+    """
+    Вычисление сходства между нарративными признаками запроса и фильма.
+    Использует те же 4 признака, что и в базе данных
+    """
+    import json
+
+    if not movie_features or not query_features:
+        return 0.0
+
+    try:
+        # Парсинг нарративных признаков фильма из JSON
+        if isinstance(movie_features, str):
+            movie_features_dict = json.loads(movie_features)
+        else:
+            movie_features_dict = movie_features
+
+        # query_features уже является словарем
+        query_features_dict = query_features
+
+        # Вычисление сходства по каждому компоненту
+        similarities = {}
+
+        # 1. Сходство ключевых слов конфликта (Jaccard similarity)
+        query_keywords = set(query_features_dict.get("conflict_keywords", []))
+        movie_keywords = set(movie_features_dict.get("conflict_keywords", []))
+
+        if query_keywords or movie_keywords:
+            intersection = len(query_keywords.intersection(movie_keywords))
+            union = len(query_keywords.union(movie_keywords))
+            similarities["keywords"] = intersection / union if union > 0 else 0.0
+        else:
+            similarities["keywords"] = 0.0
+
+        # 2. Сходство поворотов сюжета (нормализованная разность)
+        query_turns = query_features_dict.get("plot_turns", 0)
+        movie_turns = movie_features_dict.get("plot_turns", 0)
+        max_turns = max(query_turns, movie_turns, 1)  # Избегаем деления на 0
+        similarities["plot_turns"] = 1.0 - abs(query_turns - movie_turns) / max_turns
+
+        # 3. Сходство эмоциональной вариативности
+        query_sentiment_var = query_features_dict.get("sentiment_variance", 0.0)
+        movie_sentiment_var = movie_features_dict.get("sentiment_variance", 0.0)
+        max_sentiment_var = max(query_sentiment_var, movie_sentiment_var, 0.1)
+        similarities["sentiment"] = 1.0 - abs(query_sentiment_var - movie_sentiment_var) / max_sentiment_var
+
+        # 4. Сходство плотности действий
+        query_action = query_features_dict.get("action_density", 0.0)
+        movie_action = movie_features_dict.get("action_density", 0.0)
+        max_action = max(query_action, movie_action, 0.1)
+        similarities["action"] = 1.0 - abs(query_action - movie_action) / max_action
+
+        # Взвешенная комбинация сходств
+        weights = {
+            "keywords": 0.4,  # Наибольший вес для ключевых слов
+            "plot_turns": 0.25,  # Повороты сюжета важны
+            "sentiment": 0.2,  # Эмоциональная окраска
+            "action": 0.15  # Плотность действий
+        }
+
+        weighted_similarity = sum(
+            similarities[key] * weights[key]
+            for key in similarities.keys()
+        )
+
+        return weighted_similarity
+
+    except Exception as e:
+        print(f"Ошибка вычисления нарративного сходства: {e}")
+        return 0.0
+
+
+def rerank_by_narrative_features(candidates):
+    """
+    Переранжирование кандидатов с учетом нарративных признаков
+    """
+    for candidate in candidates:
+        movie_data = candidate['movie_data']
+
+        # Базовый семантический скор
+        semantic_score = candidate['semantic_score']
+        narrative_score = candidate.get('narrative_similarity', 0.0)
+
+        # Веса для различных компонентов
+        semantic_weight = 0.65      # Основной вес на семантику
+        narrative_weight = 0.25     # Нарративные признаки
+        quality_weight = 0.1        # Качественные метрики
+
+        # Бонусы за качественные метрики
+        rating_bonus = min(movie_data.get('vote_average', 0) / 10, 0.1)
+        popularity_bonus = min(np.log(movie_data.get('popularity', 1)) / 10, 0.1)
+
+        # Итоговый скор
+        candidate['final_score'] = (
+                semantic_score * semantic_weight +
+                narrative_score * narrative_weight +
+                rating_bonus + popularity_bonus
+        )
+
+        candidate['score_breakdown'] = {
+            'semantic': semantic_score,
+            'narrative': narrative_score,
+            'quality': rating_bonus,
+            'final': candidate['final_score']
+        }
+
+    # Сортировка по итоговому скору
+    return sorted(candidates, key=lambda x: x['final_score'], reverse=True)

movie_plot_search_engine.py

@@ -0,0 +1,13 @@
+from app_simplified import _run_main_app
+
+import modal
+from modal_app import app
+
+
+if __name__ == "__main__":
+    """Запуск вычислений в Modal"""
+    print("Запуск вычислений в Modal...")
+    print("Деплоим приложение...")
+    with modal.enable_output():
+        app.deploy()  # ✅ Деплоим приложение
+    _run_main_app()

requirements.txt

@@ -0,0 +1,70 @@
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.9
+aiosignal==1.3.2
+anyio==4.9.0
+attrs==25.3.0
+certifi==2025.4.26
+charset-normalizer==3.4.2
+click==8.1.8
+filelock==3.18.0
+frozenlist==1.6.2
+fsspec==2025.5.1
+grpclib==0.4.7
+h2==4.2.0
+hf-xet==1.1.3
+hpack==4.1.0
+huggingface-hub==0.32.4
+hyperframe==6.1.0
+idna==3.10
+Jinja2==3.1.6
+joblib==1.5.1
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+modal==1.0.2
+mpmath==1.3.0
+multidict==6.4.4
+networkx==3.5
+numpy==1.26.4  # Специфичная версия, совместимая с torch
+packaging==25.0
+pandas==2.2.2
+pillow==11.2.1
+propcache==0.3.1
+protobuf==6.31.1
+Pygments==2.19.1
+python-dateutil==2.9.0.post0
+pytz==2025.2
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.3
+rich==14.0.0
+safetensors==0.5.3
+scikit-learn==1.7.0
+scipy==1.15.3
+sentence-transformers==2.7.0
+shellingham==1.5.4
+sigtools==4.0.1
+six==1.17.0
+sniffio==1.3.1
+spacy==3.7.5
+sympy==1.14.0
+synchronicity==0.9.13
+textacy==0.13.0
+textblob==0.17.1
+threadpoolctl==3.6.0
+tokenizers==0.21.1
+toml==0.10.2
+torch==2.2.2
+tqdm==4.66.4
+transformers==4.52.4
+typer>=0.9
+types-certifi==2021.10.8.3
+types-toml==0.10.8.20240310
+typing_extensions==4.14.0
+tzdata==2025.2
+urllib3==2.4.0
+watchfiles==1.0.5
+yarl==1.20.0
+
+mcp[cli]>=0.1.0
+uvicorn

requirements_modal.txt

@@ -0,0 +1,81 @@
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.9
+aiosignal==1.3.2
+anyio==4.9.0
+attrs==25.3.0
+certifi==2025.4.26
+charset-normalizer==3.4.2
+click==8.1.8
+faiss-gpu-cu12>=1.8.0  # Совместим с CUDA 12.8[3]
+fastparquet>=2024.2.0
+filelock==3.18.0
+frozenlist==1.6.2
+fsspec==2025.5.1
+gradio>=4.0.0          # Для веб-интерфейса
+grpclib==0.4.7
+h2==4.2.0
+hf-xet==1.1.3
+hpack==4.1.0
+huggingface-hub==0.32.4
+hyperframe==6.1.0
+idna==3.10
+Jinja2==3.1.6
+joblib==1.5.1
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+modal==1.0.2
+mpmath==1.3.0
+multidict==6.4.4
+networkx==3.5
+numpy==1.26.4  # Специфичная версия, совместимая с torch
+openai>=1.0.0          # Для работы с Nebius API[6]
+packaging==25.0
+pandas==2.2.2
+pillow==11.2.1
+propcache==0.3.1
+protobuf==6.31.1
+pyarrow>=14.0.0
+Pygments==2.19.1
+python-dateutil==2.9.0.post0
+pytz==2025.2
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.3
+rich==14.0.0
+safetensors==0.5.3
+scikit-learn==1.7.0
+scipy==1.15.3
+sentence-transformers==2.7.0
+shellingham==1.5.4
+sigtools==4.0.1
+six==1.17.0
+sniffio==1.3.1
+spacy[cuda-autodetect]==3.7.5
+sympy==1.14.0
+synchronicity==0.9.13
+textacy==0.13.0     # Для извлечения ключевых слов
+textblob==0.17.1    # Для анализа тональности
+threadpoolctl==3.6.0
+tokenizers==0.21.1
+toml==0.10.2
+torch==2.2.2
+tqdm==4.66.4
+transformers==4.52.4
+typer>=0.9
+types-certifi==2021.10.8.3
+types-toml==0.10.8.20240310
+typing_extensions==4.14.0
+tzdata==2025.2
+urllib3==2.4.0
+watchfiles==1.0.5
+yarl==1.20.0
+# Для загрузки модели с помощью transformers
+accelerate>=0.21.0
+# Для 8-битной и 4-битной квантизации моделей
+bitsandbytes>=0.43.0
+# Для работы с pipeline и генерацией текста
+sentencepiece
+# LlamaIndex пакеты
+llama-index-core>=0.10.0
+llama-index-llms-openai>=0.1.0

setup_image.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""
+Скрипт настройки образа для PlotMatcher
+"""
+import os
+import subprocess
+import sys
+
+
+def run_command(cmd):
+    """Выполнение команды с проверкой ошибок"""
+    print(f"Выполняется: {cmd}")
+    result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(f"Ошибка: {result.stderr}")
+        sys.exit(1)
+    return result.stdout
+
+
+def main():
+    print("Настройка образа PlotMatcher...")
+
+    # Проверка CUDA
+    try:
+        output = run_command("nvidia-smi")
+        print("CUDA доступна:")
+        print(output)
+    except:
+        print("Предупреждение: nvidia-smi недоступна на этапе сборки")
+
+    # Создание необходимых директорий
+    os.makedirs("/data", exist_ok=True)
+    os.makedirs("/tmp/model_cache", exist_ok=True)
+
+    print("Настройка завершена успешно!")
+
+
+if __name__ == "__main__":
+    main()

setup_punkt_extraction.py

@@ -0,0 +1,92 @@
+# setup_punkt_extraction.py
+
+import pickle
+import os
+import ast
+import sys
+
+
+def extract_punkt_data_to_files():
+    """Извлечение данных из english.pickle в отдельные файлы"""
+
+    # Путь к pickle файлу
+    pickle_path = "/root/nltk_data/tokenizers/punkt_tab/english/english.pickle"
+    output_dir = "/root/nltk_data/tokenizers/punkt_tab/english"
+
+    try:
+        print(f"Loading punkt model from {pickle_path}")
+
+        # Загрузка модели
+        with open(pickle_path, 'rb') as f:
+            punkt_model = pickle.load(f)
+
+        print(f"Punkt model loaded successfully: {type(punkt_model)}")
+
+        # 1. Извлечение sentence starters
+        try:
+            if hasattr(punkt_model, '_lang_vars') and punkt_model._lang_vars:
+                sent_starters = punkt_model._lang_vars.sent_starters
+                with open(f"{output_dir}/sent_starters.txt", 'w') as f:
+                    f.write('\n'.join(sent_starters))
+                print(f"✅ Created sent_starters.txt with {len(sent_starters)} entries")
+            else:
+                print("⚠️ No sentence starters found, creating default ones")
+                default_starters = ["i", "you", "he", "she", "it", "we", "they", "the", "a", "an"]
+                with open(f"{output_dir}/sent_starters.txt", 'w') as f:
+                    f.write('\n'.join(default_starters))
+        except Exception as e:
+            print(f"⚠️ Error extracting sentence starters: {e}")
+            # Создаем базовые стартеры
+            default_starters = ["i", "you", "he", "she", "it", "we", "they", "the", "a", "an"]
+            with open(f"{output_dir}/sent_starters.txt", 'w') as f:
+                f.write('\n'.join(default_starters))
+
+        # 2. Извлечение collocations
+        try:
+            if hasattr(punkt_model, '_params') and punkt_model._params:
+                collocations = punkt_model._params.collocations
+                with open(f"{output_dir}/collocations.tab", 'w') as f:
+                    for (word1, word2), freq in collocations.items():
+                        f.write(f"{word1}\t{word2}\t{freq}\n")
+                print(f"✅ Created collocations.tab with {len(collocations)} entries")
+            else:
+                # Создаем пустой файл
+                open(f"{output_dir}/collocations.tab", 'w').close()
+                print("✅ Created empty collocations.tab")
+        except Exception as e:
+            print(f"⚠️ Error extracting collocations: {e}")
+            open(f"{output_dir}/collocations.tab", 'w').close()
+
+        # 3. Создание остальных файлов
+        try:
+            # Abbreviations
+            if hasattr(punkt_model, '_params') and hasattr(punkt_model._params, 'abbrev_types'):
+                with open(f"{output_dir}/abbrev_types.txt", 'w') as f:
+                    f.write('\n'.join(punkt_model._params.abbrev_types))
+                print("✅ Created abbrev_types.txt from model")
+            else:
+                # Создаем пустой файл
+                open(f"{output_dir}/abbrev_types.txt", 'w').close()
+                print("✅ Created empty abbrev_types.txt")
+
+            # Ortho context (обычно пустой)
+            open(f"{output_dir}/ortho_context.tab", 'w').close()
+            print("✅ Created empty ortho_context.tab")
+
+        except Exception as e:
+            print(f"⚠️ Warning creating additional files: {e}")
+            # Создаем пустые файлы на всякий случай
+            for filename in ["abbrev_types.txt", "ortho_context.tab"]:
+                open(f"{output_dir}/{filename}", 'w').close()
+
+        print("✅ All punkt_tab files created successfully")
+        return True
+
+    except Exception as e:
+        print(f"❌ Error extracting punkt data: {e}")
+        return False
+
+
+if __name__ == "__main__":
+    success = extract_punkt_data_to_files()
+    sys.exit(0 if success else 1)

test_mcp.py

@@ -0,0 +1,53 @@
+# test_mcp.py
+import asyncio
+from tools.client import MCPToolManager
+
+
+async def main():
+    print("🔌 Testing MCP Integration...")
+
+    manager = MCPToolManager()
+
+    # Подключаем локальный сервер поиска (он будет дергать Modal внутри)
+    # Убедитесь, что путь правильный относительно запуска
+    await manager.connect_to_server("vectordb", "tools/mcp_server_vectordb.py")
+
+    print("\n📋 Listing available tools via MCP:")
+    tools = await manager.list_tools()
+    for t in tools:
+        print(f" - {t.name}: {t.description}")
+
+    print("\n🔍 Testing search_plots tool via MCP protocol...")
+    try:
+        # Реальный вызов через MCP -> Client -> Server -> Modal
+        result = await manager.call_tool("search_plots", {
+            "query": "In a distant future, Earth has been abandoned and covered in trash. "
+                     "A small waste-collecting robot has been left behind to clean up the planet. "
+                     "He spends his days compacting garbage into cubes, but he's completely alone "
+                     "and dreams of companionship. One day, a sleek probe robot arrives from space, "
+                     "and he falls in love with her, leading to an adventure across the galaxy.",
+            "limit": 5
+        })
+        # ✅ ИСПРАВЛЕНО: Правильное извлечение содержимого из CallToolResult
+        if hasattr(result, 'content') and result.content:
+            # result.content - это список TextContent объектов
+            # Берём текст из первого элемента
+            content_text = result.content[0].text
+            print(f"✅ Result received:\n{content_text}")
+        else:
+            # Fallback на случай неожиданного формата
+            print(f"✅ Result received: {result}")
+
+        # print(f"✅ Result received: {result[:200]}...")  # Печатаем начало JSON
+    except Exception as e:
+        import traceback
+        print(f"❌ Error: {e}")
+        traceback.print_exc()
+
+    finally:
+        await manager.cleanup()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+    

tools/client.py

@@ -0,0 +1,102 @@
+# tools/client.py
+import asyncio
+from contextlib import AsyncExitStack
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+import logging
+from typing import List, Dict, Any
+
+# Настройка логирования
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("mcp-client")
+
+
+class MCPToolManager:
+    """
+    Безопасный клиент для управления инструментами MCP.
+    Реализует Explicit Allowlist для защиты от инъекций возможностей.
+    """
+
+    def __init__(self):
+        self.exit_stack = AsyncExitStack()
+        self.sessions = {}
+
+        # --- EXPLICIT ALLOWLIST ---
+        # Только эти инструменты будут доступны агентам.
+        # Любые другие инструменты, предлагаемые серверами, будут игнорироваться.
+        self.ALLOWED_TOOLS = {
+            "search_plots",  # из vectordb
+            "get_movie_details",  # из tmdb
+            # "get_credits"       # можно раскомментировать, если нужно
+        }
+
+    async def connect_to_server(self, server_name: str, script_path: str):
+        """Подключается к локальному Python-скрипту как к MCP серверу"""
+        # Используем python для запуска скрипта сервера
+        server_params = StdioServerParameters(
+            command="python",
+            args=[script_path],
+            env=None
+        )
+
+        try:
+            # Инициализация транспорта и сессии
+            stdio_transport = await self.exit_stack.enter_async_context(stdio_client(server_params))
+            read, write = stdio_transport
+            session = await self.exit_stack.enter_async_context(ClientSession(read, write))
+            await session.initialize()
+
+            self.sessions[server_name] = session
+            logger.info(f"✅ Connected to MCP server: {server_name}")
+
+            # Валидация и логирование доступных инструментов
+            await self._validate_tools(server_name, session)
+
+        except Exception as e:
+            logger.error(f"❌ Failed to connect to {server_name} ({script_path}): {e}")
+
+    async def _validate_tools(self, server_name: str, session: ClientSession):
+        """Проверяет инструменты сервера на соответствие белому списку"""
+        tools_response = await session.list_tools()
+
+        allowed_count = 0
+        blocked_count = 0
+
+        for tool in tools_response.tools:
+            if tool.name in self.ALLOWED_TOOLS:
+                allowed_count += 1
+            else:
+                blocked_count += 1
+                logger.warning(f"⚠️ BLOCKED tool '{tool.name}' from {server_name} (not in allowlist)")
+
+        logger.info(f"Server {server_name}: {allowed_count} allowed, {blocked_count} blocked.")
+
+    async def list_tools(self) -> List[Any]:
+        """Возвращает список всех РАЗРЕШЕННЫХ инструментов со всех серверов"""
+        all_tools = []
+        for session in self.sessions.values():
+            result = await session.list_tools()
+            # Фильтруем на лету
+            valid_tools = [t for t in result.tools if t.name in self.ALLOWED_TOOLS]
+            all_tools.extend(valid_tools)
+        return all_tools
+
+    async def call_tool(self, tool_name: str, arguments: dict) -> Any:
+        """Вызов инструмента на нужном сервере"""
+        # Дополнительная проверка перед вызовом
+        if tool_name not in self.ALLOWED_TOOLS:
+            raise SecurityError(f"Security Alert: Attempt to call unauthorized tool '{tool_name}'")
+
+        for name, session in self.sessions.items():
+            # Проверяем, есть ли инструмент на этом сервере
+            tools = await session.list_tools()
+            if any(t.name == tool_name for t in tools.tools):
+                logger.info(f"📞 Calling {tool_name} on {name} with args: {arguments}")
+                result = await session.call_tool(tool_name, arguments)
+                return result
+
+        raise ValueError(f"Tool {tool_name} not found on any connected server")
+
+    async def cleanup(self):
+        """Закрытие всех соединений"""
+        await self.exit_stack.aclose()

tools/mcp_server_tmdb.py

@@ -0,0 +1,104 @@
+# tools/mcp_server_tmdb.py
+from mcp.server.fastmcp import FastMCP
+import pandas as pd
+import json
+import os
+import logging
+
+# Настройка логирования
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("mcp-tmdb")
+
+# Инициализация MCP сервера
+mcp = FastMCP("TMDB_Local_Data")
+
+# Глобальная переменная для DataFrame
+# Мы используем lazy loading, чтобы не грузить память при старте, если не нужно
+MOVIES_DF = None
+METADATA_PATH = "data/indexed_movies_metadata.parquet"  # Путь, куда modal_app.py сохранил метаданные
+
+
+def _get_movies_df():
+    """Ленивая загрузка и кэширование датафрейма"""
+    global MOVIES_DF
+    if MOVIES_DF is None:
+        if not os.path.exists(METADATA_PATH):
+            # Если запускаемся локально и файла нет, можно попробовать скачать его с Modal Volume
+            # Но для простоты предполагаем, что он есть (или смонтирован)
+            logger.error(f"Metadata file not found at {METADATA_PATH}")
+            return None
+
+        logger.info(f"Loading metadata from {METADATA_PATH}...")
+        try:
+            df = pd.read_parquet(METADATA_PATH)
+            # Создаем индекс по ID для мгновенного поиска O(1)
+            # Убедимся, что ID - это int (иногда бывают строки)
+            if 'id' in df.columns:
+                df['id'] = df['id'].astype(int)
+                df.set_index('id', inplace=True)
+            MOVIES_DF = df
+            logger.info(f"Loaded {len(df)} movies into memory.")
+        except Exception as e:
+            logger.error(f"Failed to load metadata: {e}")
+            return None
+    return MOVIES_DF
+
+
+@mcp.tool()
+def get_movie_details(movie_id: int) -> str:
+    """
+    Get enriched metadata for a movie by its TMDB ID.
+    Use this to get accurate Cast, Director, Ratings, and Runtime for the final recommendation card.
+
+    Args:
+        movie_id: The integer ID of the movie (from search results).
+    """
+    df = _get_movies_df()
+    if df is None:
+        return json.dumps({"error": "Metadata service unavailable"})
+
+    try:
+        # Поиск по индексу (очень быстрый)
+        if movie_id not in df.index:
+            return json.dumps({"error": f"Movie ID {movie_id} not found in local database"})
+
+        movie = df.loc[movie_id]
+
+        # Формируем богатый ответ.
+        # Используем .get(), так как не все поля могут быть в Parquet
+        details = {
+            "id": movie_id,
+            "title": str(movie.get("title", "Unknown")),
+            "original_title": str(movie.get("original_title", "")),
+            "overview": str(movie.get("overview", "")),  # Полный текст, если в поиске был обрезан
+            "genres": str(movie.get("genres", "Unknown")),
+            "director": str(movie.get("director", "Unknown")),  # Если вы сохраняли это поле
+            "cast": str(movie.get("cast", "Unknown")),  # Если вы сохраняли это поле
+            "vote_average": float(movie.get("vote_average", 0.0)),
+            "release_date": str(movie.get("release_date", "N/A")),
+            "runtime": int(movie.get("runtime", 0)),
+            # Нарративные признаки тоже можно вернуть, если Эксперту нужно объяснить "почему"
+            "narrative_features": str(movie.get("narrative_features", "{}"))
+        }
+
+        return json.dumps(details, ensure_ascii=False)
+
+    except Exception as e:
+        logger.error(f"Error fetching details for {movie_id}: {e}")
+        return json.dumps({"error": str(e)})
+
+
+@mcp.tool()
+def get_credits(movie_id: int) -> str:
+    """
+    Get specifically the cast and director (if not available in get_movie_details).
+    Legacy tool support.
+    """
+    # В нашей архитектуре это избыточно, если get_movie_details возвращает всё.
+    # Но оставим как алиас.
+    return get_movie_details(movie_id)
+
+
+if __name__ == "__main__":
+    # Для локального запуска
+    mcp.run()

tools/mcp_server_vectordb.py

@@ -0,0 +1,106 @@
+# tools/mcp_server_vectordb.py
+from mcp.server.fastmcp import FastMCP
+import modal
+import logging
+import json
+from datetime import date
+
+# Настройка логирования
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("mcp-vectordb")
+
+# Создаем MCP сервер
+mcp = FastMCP("MovieVectorDB")
+
+# Подключение к Modal (инициализируется один раз)
+try:
+    encode_func = modal.Function.from_name("tmdb-project", "encode_user_query")
+    search_func = modal.Function.from_name("tmdb-project", "search_similar_movies")
+    logger.info("✅ Connected to Modal search functions")
+except Exception as e:
+    logger.error(f"❌ Failed to connect to Modal: {e}")
+    encode_func = None
+    search_func = None
+
+
+@mcp.tool()
+def search_plots(query: str, filters: str = None, limit: int = 5) -> str:
+    """
+    Semantic search for movies based on a plot description.
+
+    Args:
+        query: Detailed description of the movie plot, theme, or story (e.g. "A robot learns to love").
+        filters: JSON string with narrative filters (optional). Example: '{"min_action": 0.5}'.
+        limit: Number of movies to return (default 5, max 20).
+
+    Returns:
+        JSON string containing a list of found movies with their IDs, titles, and relevance scores.
+    """
+    if not encode_func or not search_func:
+        return json.dumps({"error": "Search service unavailable"})
+
+    logger.info(f"Searching for: {query[:50]}...")
+
+    try:
+        # 1. Векторизация (используем логику из modal_app.py)
+        # remove_entities=True - важно для поиска по смыслу
+        encoding_result = encode_func.remote(query, remove_entities=True)
+        # ✅ ИЗМЕНЕНО: Отключаем remove_entities для сохранения всех слов
+        # encoding_result = encode_func.remote(query, remove_entities=False)
+
+        # 2. Поиск
+        # Преобразуем фильтры из строки, если они есть
+        narrative_params = encoding_result["narrative_features"]
+        if filters:
+            try:
+                extra_filters = json.loads(filters)
+                narrative_params.update(extra_filters)
+            except:
+                pass
+
+        results_dict = search_func.remote(
+            query_embedding=encoding_result["embedding"],
+            query_narrative_features=narrative_params,
+            top_k=limit * 2,  # Берем с запасом
+            rerank_top_n=limit
+        )
+
+        candidates = results_dict.get("results", [])
+
+        # Возвращаем упрощенный список для LLM (экономия токенов)
+        simplified_results = []
+        for c in candidates:
+            movie = c.get("movie_data", {})
+
+            release_date = movie.get("release_date")
+            year = "N/A"
+
+            if release_date:
+                if isinstance(release_date, str):
+                    # Если строка - берём первые 4 символа
+                    year = release_date[:4] if len(release_date) >= 4 else release_date
+                elif isinstance(release_date, date):
+                    # Если datetime.date объект - используем .year
+                    year = str(release_date.year)
+                else:
+                    # Пробуем преобразовать к строке
+                    year = str(release_date)[:4]
+            simplified_results.append({
+                "id": movie.get("id"),
+                "title": movie.get("title"),
+                "relevance_score": c.get("final_score", 0.0),
+                "year": year,
+                "overview": movie.get("overview", "")[:150] + "..."
+            })
+
+        return json.dumps(simplified_results, indent=2)
+
+    except Exception as e:
+        import traceback
+        logger.error(f"Search error: {e}")
+        traceback.print_exc()
+        return json.dumps({"error": str(e)})
+
+
+if __name__ == "__main__":
+    mcp.run()