api.py

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
import pandas as pd
import numpy as np
import tensorflow as tf
from yahoo_fin.stock_info import get_data
from sklearn.preprocessing import MinMaxScaler
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from pytorch_forecasting import TemporalFusionTransformer
from bs4 import BeautifulSoup
import requests
from dotenv import load_dotenv
import os
from fastapi.middleware.cors import CORSMiddleware

os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

import torch

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

MODEL_PATH = "lib/20_lstm_model.h5"
model = tf.keras.models.load_model(MODEL_PATH)

model_name_news = "mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis"
tokenizer = AutoTokenizer.from_pretrained(model_name_news)
sentiment_model = AutoModelForSequenceClassification.from_pretrained(
    model_name_news).to(device)

best_model_path = 'lib/tft_pred.ckpt'

best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path).to(device)

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["GET", "POST", "PUT", "DELETE"],
    allow_headers=["*"],
)


class TickerRequest(BaseModel):
    ticker: str
    start_date: str
    end_date: str
    interval: str = "1d"

def fetch_and_process_ticker_data(ticker, start_date, end_date, interval="1d"):
    df = pd.DataFrame()
    try:
        temp = get_data(ticker, start_date=start_date,
                        end_date=end_date, index_as_date=True, interval=interval)
        temp = temp.drop(columns="close")
        temp["revenue"] = temp["adjclose"] * temp["volume"]
        temp["daily_profit"] = temp["adjclose"] - temp["open"]
        df = pd.concat([df, temp], axis=0)

    except Exception as error:
        raise HTTPException(
            status_code=500, detail=f"Error processing ticker {ticker}: {error}")
    return df


def ticker_encoded(df):
    label_map = {'ATOM': 0, 'HBIO': 1, 'IBEX': 2, 'MYFW': 3, 'NATH': 4}

    ticker_encoded = []

    for i in df.iloc():

        ticker_name = i['ticker']

        encoded_ticker = label_map[ticker_name]

        ticker_encoded.append(encoded_ticker)
    df['ticker_encoded'] = ticker_encoded

    return df


def normalize(df):
    price_scaler = MinMaxScaler()
    volume_revenue_scaler = MinMaxScaler()
    profit_scaler = MinMaxScaler()

    df[["open", "high", "low", "adjclose"]] = price_scaler.fit_transform(
        df[["open", "high", "low", "adjclose"]])
    df[["volume", "revenue"]] = volume_revenue_scaler.fit_transform(
        df[["volume", "revenue"]])
    df[["daily_profit"]] = profit_scaler.fit_transform(df[["daily_profit"]])

    return df, price_scaler


def create_sequence(dataset):
    sequences = []
    labels = []
    dates = []
    stock = []

    df_copy = dataset.drop(columns=["date"])

    start_idx = 0
    for stop_idx in range(20, len(dataset)):
        set_ = set(dataset.iloc[start_idx:stop_idx]["ticker_encoded"].values)
        target_day_ticker_class = dataset.iloc[stop_idx]["ticker_encoded"]

        if len(set_) == 1 and target_day_ticker_class == list(set_)[0]:
            sequences.append(df_copy.iloc[start_idx:stop_idx].values)
            labels.append(df_copy.iloc[stop_idx][["open", "adjclose"]])
            date_string = dataset.iloc[stop_idx]["date"].strftime('%Y-%m-%d')
            dates.append(date_string)
            stock.append(str(dataset.iloc[stop_idx]["ticker_encoded"]))

        start_idx += 1

    return np.array(sequences), np.array(labels), dates, stock


def scaling_predictions(price_scaler, combined_dataset_prediction):

    price_scaler.min_ = np.array([price_scaler.min_[0], price_scaler.min_[3]])

    price_scaler.scale_ = np.array(
        [price_scaler.scale_[0], price_scaler.scale_[3]])

    combined_dataset_prediction_inverse = price_scaler.inverse_transform(
        combined_dataset_prediction)

    return combined_dataset_prediction_inverse


def storing_predictions(df, dates, stock, combined_dataset_prediction_inverse):

    df['pred_open'] = np.nan

    df['pred_closing'] = np.nan

    for idx, row in df.iterrows():

        current_row_date = row.date.strftime('%Y-%m-%d')

        current_row_ticker = str(row.ticker_encoded)

        for i in range(len(dates)):

            if current_row_date == dates[i] and stock[i] == current_row_ticker:

                opening_price = combined_dataset_prediction_inverse[i][0]
                closing_price = combined_dataset_prediction_inverse[i][1]
                df.loc[idx, 'pred_open'] = opening_price
                df.loc[idx, 'pred_closing'] = closing_price

                break
    df = df.dropna(subset=['pred_open', 'pred_closing']).reset_index(drop=True)

    return df


def scrape_news(ticker_name):

    columns = ['datatime', 'title', 'source',
               'link', 'top_sentiment', 'sentiment_score']
    df = pd.DataFrame(columns=columns)

    for i in range(1, 3):

        url = f'https://markets.businessinsider.com/news/{ticker_name}-stock?p={i}'
        response = requests.get(url)
        html = response.text
        soup = BeautifulSoup(html, 'lxml')

        articles = soup.find_all('div', class_='latest-news__story')

        for article in articles:
            datatime = article.find(
                'time', class_='latest-news__date').get('datetime')

            title = article.find('a', class_='news-link').text

            source = article.find('span', class_='latest-news__source').text

            link = article.find('a', class_='news-link').get('href')

            top_sentiment = ''

            sentiment_score = 0

            temp = pd.DataFrame(
                [[datatime, title, source, link, top_sentiment, sentiment_score]], columns=df.columns)

            df = pd.concat([temp, df], axis=0)

    return df


def add_recent_news(main_df, news_df, lookback_days=10):

    news_df.drop(columns=['top_sentiment', 'sentiment_score'], inplace=True)

    main_df['date'] = pd.to_datetime(main_df['date'])
    news_df['datatime'] = pd.to_datetime(news_df['datatime'])

    news_list = []
    last_available_news = ''

    for _, row in main_df.iterrows():
        current_date = row['date']
        current_ticker = row['ticker']
        news_articles = ''

        for _, news_row in news_df.iterrows():
            extracted_date = news_row['datatime']

            if (current_date - extracted_date).days <= lookback_days and extracted_date < current_date:
                news_articles += news_row['title'] + " "

        if not news_articles.strip():
            for _, news_row in news_df[::-1].iterrows():
                if news_row['datatime'] < current_date:
                    news_articles = news_row['title']
                    break

        last_available_news = news_articles.strip() or last_available_news
        news_list.append(last_available_news)

    main_df['news'] = news_list

    return main_df


def news_sentiment(df):

    news_column_name = 'news'
    texts = df[news_column_name].tolist()

    inputs = tokenizer(texts, padding=True,
                       truncation=True, return_tensors="pt")
    inputs = {key: val.to(device) for key, val in inputs.items()} 


    with torch.no_grad():
        outputs = sentiment_model(**inputs)

    logits = outputs.logits
    probs = torch.softmax(logits, dim=-1)

    labels = ["negative", "neutral", "positive"]

    predictions = torch.argmax(probs, dim=-1)

    df['predicted_sentiment'] = pd.Series(
        [labels[pred] for pred in predictions], index=df[df[news_column_name].notna()].index)

    sentiment_map = {
        'positive': 1,
        'neutral': 0,
        'negative': -1
    }

    df['sentiment_score'] = df['predicted_sentiment'].map(sentiment_map)

    df = df.drop(columns=['news'])

    return df


def get_tft_predictions(df):
    for i in range(1, 21):
        df[f'open_lag_{i}'] = df.groupby('ticker')['open'].shift(i)
        df[f'adjclose_lag_{i}'] = df.groupby('ticker')['adjclose'].shift(i)

    lag_columns = [f'open_lag_{i}' for i in range(
        1, 21)] + [f'adjclose_lag_{i}' for i in range(1, 21)]

    df.dropna(subset=lag_columns, inplace=True)

    predictions = best_tft.predict(df.to(device), mode="quantiles")

    return predictions


@app.get("/fetch-ticker-data/{ticker_name}/{start_date}/{end_date}/{interval}")
async def fetch_ticker_data(ticker_name: str, start_date: str, end_date: str, interval: str):
    try:
        result_df = fetch_and_process_ticker_data(
            ticker=ticker_name,
            start_date=start_date,
            end_date=end_date,
            interval=interval
        )
        return result_df.to_dict(orient="records")
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))


@app.get("/predict-prices/{ticker_name}/{start_date}/{end_date}/{interval}")
async def predict_prices(ticker_name: str, start_date: str, end_date: str, interval: str):
    try:
        result_df = fetch_and_process_ticker_data(
            ticker=ticker_name,
            start_date=start_date,
            end_date=end_date,
            interval=interval
        )

        raw_data = result_df.tail(60)
        raw_data = raw_data.reset_index()

        raw_data.rename(columns={"index": "date"}, inplace=True)
        raw_data = ticker_encoded(raw_data)

        temp_df = raw_data.copy()

        normalized_data, scaler = normalize(raw_data)
        normalized_data = normalized_data.drop(columns=['ticker'])

        sequences, _, dates, stock = create_sequence(normalized_data)
        combined_dataset_prediction = model.predict(sequences)
        combined_dataset_prediction_inverse = scaling_predictions(
            scaler, combined_dataset_prediction)

        lstm_pred_df = storing_predictions(
            temp_df, dates, stock, combined_dataset_prediction_inverse)
        news_df = scrape_news(ticker_name=ticker_name)

        combined_with_news_df = add_recent_news(lstm_pred_df, news_df)
        sentiment_df = news_sentiment(combined_with_news_df)

        sentiment_df['time_idx'] = range(1, len(sentiment_df) + 1)

        predicted_values = get_tft_predictions(sentiment_df)

        final_pred_open_price = predicted_values[0].item()
        final_pred_closing_price = predicted_values[1].item()

        return {"open": final_pred_open_price, 'close': final_pred_closing_price}

    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))