Upload 11 files

pages/10_Optimized_Result_Analysis.py

@@ -0,0 +1,399 @@
+import streamlit as st
+from numerize.numerize import numerize
+import pandas as pd
+from utilities import (format_numbers,decimal_formater,
+                       load_local_css,set_header,
+                       initialize_data,
+                       load_authenticator)
+import pickle
+import streamlit_authenticator as stauth
+import yaml
+from yaml import SafeLoader
+from classes import class_from_dict
+import plotly.express as px 
+import numpy as np
+import plotly.graph_objects as go
+import pandas as pd
+
+
+def summary_plot(data, x, y, title, text_column, color, format_as_percent=False, format_as_decimal=False):
+    fig = px.bar(data, x=x, y=y, orientation='h',
+                 title=title, text=text_column, color=color)
+    fig.update_layout(showlegend=False)
+    data[text_column] = pd.to_numeric(data[text_column], errors='coerce')
+    
+    # Update the format of the displayed text based on the chosen format
+    if format_as_percent:
+        fig.update_traces(texttemplate='%{text:.0%}', textposition='outside', hovertemplate='%{x:.0%}')
+    elif format_as_decimal:
+        fig.update_traces(texttemplate='%{text:.2f}', textposition='outside', hovertemplate='%{x:.2f}')
+    else:
+        fig.update_traces(texttemplate='%{text:.2s}', textposition='outside', hovertemplate='%{x:.2s}')
+    
+    fig.update_layout(xaxis_title=x, yaxis_title='Channel Name', showlegend=False)
+    return fig
+
+
+def stacked_summary_plot(data, x, y, title, text_column, color_column, stack_column=None, format_as_percent=False, format_as_decimal=False):
+    fig = px.bar(data, x=x, y=y, orientation='h',
+                 title=title, text=text_column, color=color_column, facet_col=stack_column)
+    fig.update_layout(showlegend=False)
+    data[text_column] = pd.to_numeric(data[text_column], errors='coerce')
+
+    # Update the format of the displayed text based on the chosen format
+    if format_as_percent:
+        fig.update_traces(texttemplate='%{text:.0%}', textposition='outside', hovertemplate='%{x:.0%}')
+    elif format_as_decimal:
+        fig.update_traces(texttemplate='%{text:.2f}', textposition='outside', hovertemplate='%{x:.2f}')
+    else:
+        fig.update_traces(texttemplate='%{text:.2s}', textposition='outside', hovertemplate='%{x:.2s}')
+
+    fig.update_layout(xaxis_title=x, yaxis_title='', showlegend=False)
+    return fig
+
+
+
+def funnel_plot(data, x, y, title, text_column, color_column, format_as_percent=False, format_as_decimal=False):
+    data[text_column] = pd.to_numeric(data[text_column], errors='coerce')
+
+    # Round the numeric values in the text column to two decimal points
+    data[text_column] = data[text_column].round(2)
+
+    # Create a color map for categorical data
+    color_map = {category: f'rgb({i * 30 % 255},{i * 50 % 255},{i * 70 % 255})' for i, category in enumerate(data[color_column].unique())}
+    
+    fig = go.Figure(go.Funnel(
+        y=data[y],
+        x=data[x],
+        text=data[text_column],
+        marker=dict(color=data[color_column].map(color_map)),
+        textinfo="value",
+        hoverinfo='y+x+text'
+    ))
+
+    # Update the format of the displayed text based on the chosen format
+    if format_as_percent:
+        fig.update_layout(title=title, funnelmode="percent")
+    elif format_as_decimal:
+        fig.update_layout(title=title, funnelmode="overlay")
+    else:
+        fig.update_layout(title=title, funnelmode="group")
+
+    return fig
+
+
+st.set_page_config(layout='wide')
+load_local_css('styles.css')
+set_header()
+
+# for k, v in st.session_state.items():
+#     if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+#         st.session_state[k] = v
+
+st.empty()
+st.header('Model Result Analysis')
+spends_data=pd.read_excel('Overview_data_test.xlsx')
+
+with open('summary_df.pkl', 'rb') as file:
+  summary_df_sorted = pickle.load(file)
+
+selected_scenario= st.selectbox('Select Saved Scenarios',['S1','S2']) 
+
+st.header('Optimized Spends Overview')
+___columns=st.columns(3)
+with ___columns[2]:
+    fig=summary_plot(summary_df_sorted, x='Delta_percent', y='Channel_name', title='Delta', text_column='Delta_percent',color='Channel_name')
+    st.plotly_chart(fig,use_container_width=True)
+with ___columns[0]:
+    fig=summary_plot(summary_df_sorted, x='Actual_spend', y='Channel_name', title='Actual Spend', text_column='Actual_spend',color='Channel_name')
+    st.plotly_chart(fig,use_container_width=True)         
+with ___columns[1]:
+    fig=summary_plot(summary_df_sorted, x='Optimized_spend', y='Channel_name', title='Planned Spend', text_column='Optimized_spend',color='Channel_name')
+    st.plotly_chart(fig,use_container_width=False)
+
+st.header(' Budget Allocation')
+summary_df_sorted['Perc_alloted']=np.round(summary_df_sorted['Optimized_spend']/summary_df_sorted['Optimized_spend'].sum(),2)
+columns2=st.columns(2)
+with columns2[0]:
+    fig=summary_plot(summary_df_sorted, x='Optimized_spend', y='Channel_name', title='Planned Spend', text_column='Optimized_spend',color='Channel_name')
+    st.plotly_chart(fig,use_container_width=True)
+with columns2[1]:
+    fig=summary_plot(summary_df_sorted, x='Perc_alloted', y='Channel_name', title='% Split', text_column='Perc_alloted',color='Channel_name',format_as_percent=True)
+    st.plotly_chart(fig,use_container_width=True)
+
+
+if 'raw_data' not in st.session_state:
+    st.session_state['raw_data']=pd.read_excel('raw_data_nov7_combined1.xlsx')
+    st.session_state['raw_data']=st.session_state['raw_data'][st.session_state['raw_data']['MediaChannelName'].isin(summary_df_sorted['Channel_name'].unique())] 
+    st.session_state['raw_data']=st.session_state['raw_data'][st.session_state['raw_data']['Date'].isin(spends_data["Date"].unique())]
+
+
+
+#st.write(st.session_state['raw_data']['ResponseMetricName']) 
+# st.write(st.session_state['raw_data'])
+
+
+st.header('Response Forecast Overview')
+raw_data=st.session_state['raw_data']
+effectiveness_overall=raw_data.groupby('ResponseMetricName').agg({'ResponseMetricValue': 'sum'}).reset_index()
+effectiveness_overall['Efficiency']=effectiveness_overall['ResponseMetricValue'].map(lambda x: x/raw_data['Media Spend'].sum() )
+# st.write(effectiveness_overall)
+
+columns6=st.columns(3)
+
+effectiveness_overall.sort_values(by=['ResponseMetricValue'],ascending=False,inplace=True)
+effectiveness_overall=np.round(effectiveness_overall,2)
+effectiveness_overall['ResponseMetric'] = effectiveness_overall['ResponseMetricName'].apply(lambda x: 'BAU' if 'BAU' in x else ('Gamified' if 'Gamified' in x else x))
+# effectiveness_overall=np.where(effectiveness_overall[effectiveness_overall['ResponseMetricName']=="Adjusted Account Approval BAU"],"Adjusted Account Approval BAU",effectiveness_overall['ResponseMetricName'])
+
+effectiveness_overall.replace({'ResponseMetricName':{'BAU approved clients - Appsflyer':'Approved clients - Appsflyer',
+                                                     'Gamified approved clients - Appsflyer':'Approved clients - Appsflyer'}},inplace=True)
+
+# st.write(effectiveness_overall.sort_values(by=['ResponseMetricValue'],ascending=False))
+
+
+condition = effectiveness_overall['ResponseMetricName'] == "Adjusted Account Approval BAU"
+condition1= effectiveness_overall['ResponseMetricName'] == "Approved clients - Appsflyer"
+effectiveness_overall['ResponseMetric'] = np.where(condition, "Adjusted Account Approval BAU", effectiveness_overall['ResponseMetric'])
+
+effectiveness_overall['ResponseMetricName'] = np.where(condition1, "Approved clients - Appsflyer (BAU, Gamified)", effectiveness_overall['ResponseMetricName'])
+# effectiveness_overall=pd.DataFrame({'ResponseMetricName':["App Installs - Appsflyer",'Account Requests - Appsflyer',
+#                                                           'Total Adjusted Account Approval','Adjusted Account Approval BAU',
+#                                                           'Approved clients - Appsflyer','Approved clients - Appsflyer'],
+#                                     'ResponseMetricValue':[683067,367020,112315,79768,36661,16834],
+#                                     'Efficiency':[1.24,0.67,0.2,0.14,0.07,0.03],
+custom_colors = {
+    'App Installs - Appsflyer': 'rgb(255, 135, 0)',       # Steel Blue (Blue)
+    'Account Requests - Appsflyer': 'rgb(125, 239, 161)',  # Cornflower Blue (Blue)
+    'Adjusted Account Approval': 'rgb(129, 200, 255)',      # Dodger Blue (Blue)
+    'Adjusted Account Approval BAU': 'rgb(255, 207, 98)',  # Light Sky Blue (Blue)
+    'Approved clients - Appsflyer': 'rgb(0, 97, 198)',  # Light Blue (Blue)
+    "BAU": 'rgb(41, 176, 157)',                              # Steel Blue (Blue)
+     "Gamified": 'rgb(213, 218, 229)'                      # Silver (Gray)
+    # Add more categories and their respective shades of blue as needed
+}
+
+
+
+
+
+
+with columns6[0]:
+    revenue=(effectiveness_overall[effectiveness_overall['ResponseMetricName']=='Total Approved Accounts - Revenue']['ResponseMetricValue']).iloc[0]
+    revenue=round(revenue / 1_000_000, 2)
+
+#     st.metric('Total Revenue', f"${revenue} M")
+# with columns6[1]:
+#     BAU=(effectiveness_overall[effectiveness_overall['ResponseMetricName']=='BAU approved clients - Revenue']['ResponseMetricValue']).iloc[0]
+#     BAU=round(BAU / 1_000_000, 2)
+#     st.metric('BAU approved clients - Revenue', f"${BAU} M")
+# with columns6[2]:
+#     Gam=(effectiveness_overall[effectiveness_overall['ResponseMetricName']=='Gamified approved clients - Revenue']['ResponseMetricValue']).iloc[0]
+#     Gam=round(Gam / 1_000_000, 2)
+#     st.metric('Gamified approved clients - Revenue', f"${Gam} M")
+
+# st.write(effectiveness_overall)
+data = {'Revenue': ['BAU approved clients - Revenue', 'Gamified approved clients- Revenue'],
+        'ResponseMetricValue': [70200000, 1770000],
+        'Efficiency':[127.54,3.21]}
+df = pd.DataFrame(data)
+
+
+columns9=st.columns([0.60,0.40])
+with columns9[0]:
+    figd = px.pie(df, 
+              names='Revenue', 
+              values='ResponseMetricValue',
+              hole=0.3,  # set the size of the hole in the donut
+              title='Effectiveness')
+    figd.update_layout(
+        margin=dict(l=0, r=0, b=0, t=0),width=100, height=180,legend=dict(
+        orientation='v',  # set orientation to horizontal
+        x=0,  # set x to 0 to move to the left
+        y=0.8  # adjust y as needed
+    )
+    )
+
+    st.plotly_chart(figd, use_container_width=True)
+
+with columns9[1]:
+    figd1 = px.pie(df, 
+              names='Revenue', 
+              values='Efficiency',
+              hole=0.3,  # set the size of the hole in the donut
+              title='Efficiency')
+    figd1.update_layout(
+    margin=dict(l=0, r=0, b=0, t=0),width=100,height=180,showlegend=False
+)
+    st.plotly_chart(figd1, use_container_width=True)
+
+effectiveness_overall['Response Metric Name']=effectiveness_overall['ResponseMetricName']
+
+
+
+columns4= st.columns([0.55,0.45])
+with columns4[0]:
+    fig=px.funnel(effectiveness_overall[~(effectiveness_overall['ResponseMetricName'].isin(['Total Approved Accounts - Revenue',
+                                                                                          'BAU approved clients - Revenue',
+                                                                                          'Gamified approved clients - Revenue',
+                                                                                          "Total Approved Accounts - Appsflyer"]))],
+                                                                                            x='ResponseMetricValue', y='Response Metric Name',color='ResponseMetric',
+                                                                                            color_discrete_map=custom_colors,title='Effectiveness',
+                                                                                            labels=None)
+    custom_y_labels=['App Installs - Appsflyer','Account Requests - Appsflyer','Adjusted Account Approval','Adjusted Account Approval BAU',
+                     "Approved clients - Appsflyer (BAU, Gamified)"
+                     ]
+    fig.update_layout(showlegend=False,
+    yaxis=dict(
+        tickmode='array',
+        ticktext=custom_y_labels,
+        )
+        )
+    fig.update_traces(textinfo='value', textposition='inside', texttemplate='%{x:.2s} ', hoverinfo='y+x+percent initial')
+
+    last_trace_index = len(fig.data) - 1
+    fig.update_traces(marker=dict(line=dict(color='black', width=2)), selector=dict(marker=dict(color='blue')))
+
+    st.plotly_chart(fig,use_container_width=True)
+
+
+
+
+
+with columns4[1]:
+
+# Your existing code for creating the bar chart
+    fig1 = px.bar((effectiveness_overall[~(effectiveness_overall['ResponseMetricName'].isin(['Total Approved Accounts - Revenue',
+                                                                                            'BAU approved clients - Revenue',
+                                                                                            'Gamified approved clients - Revenue',
+                                                                                            "Total Approved Accounts - Appsflyer"]))]).sort_values(by='ResponseMetricValue'),
+                x='Efficiency', y='Response Metric Name',
+                color_discrete_map=custom_colors, color='ResponseMetric',
+                labels=None,text_auto=True,title='Efficiency'
+                )
+
+    # Update layout and traces
+    fig1.update_traces(customdata=effectiveness_overall['Efficiency'],
+                   textposition='auto')
+    fig1.update_layout(showlegend=False) 
+    fig1.update_yaxes(title='',showticklabels=False)
+    fig1.update_xaxes(title='',showticklabels=False)
+    fig1.update_xaxes(tickfont=dict(size=20))
+    fig1.update_yaxes(tickfont=dict(size=20))
+    st.plotly_chart(fig1, use_container_width=True)
+
+
+effectiveness_overall_revenue=pd.DataFrame({'ResponseMetricName':['Approved Clients','Approved Clients'],
+                                            'ResponseMetricValue':[70201070,1768900],
+                                            'Efficiency':[127.54,3.21],
+                                            'ResponseMetric':['BAU','Gamified']
+                                            })
+# from plotly.subplots import make_subplots
+# fig = make_subplots(rows=1, cols=2, 
+#                     subplot_titles=["Effectiveness", "Efficiency"])
+
+# # Add first plot as subplot
+# fig.add_trace(go.Funnel(
+#     x = fig.data[0].x,
+#     y = fig.data[0].y,
+#     textinfo = 'value+percent initial',
+#     hoverinfo = 'x+y+percent initial'
+# ), row=1, col=1)
+
+# # Update layout for first subplot
+# fig.update_xaxes(title_text="Response Metric Value", row=1, col=1) 
+# fig.update_yaxes(ticktext = custom_y_labels, row=1, col=1)
+
+# # Add second plot as subplot
+# fig.add_trace(go.Bar(
+#     x = fig1.data[0].x, 
+#     y = fig1.data[0].y,
+#     customdata = fig1.data[0].customdata, 
+#     textposition = 'auto'
+# ), row=1, col=2)
+
+# # Update layout for second subplot
+# fig.update_xaxes(title_text="Efficiency", showticklabels=False, row=1, col=2)
+# fig.update_yaxes(title='', showticklabels=False, row=1, col=2)
+
+# fig.update_layout(height=600, width=800, title_text="Key Metrics")
+# st.plotly_chart(fig)
+
+
+st.header('Return Forecast by Media Channel')
+with st.expander("Return Forecast by Media Channel"):
+    metric_data=[val for val in list(st.session_state['raw_data']['ResponseMetricName'].unique()) if val!=np.NaN] 
+    # st.write(metric_data)
+    metric=st.selectbox('Select Metric',metric_data,index=1)
+
+    selected_metric=st.session_state['raw_data'][st.session_state['raw_data']['ResponseMetricName']==metric]
+    # st.dataframe(selected_metric.head(2))
+    selected_metric=st.session_state['raw_data'][st.session_state['raw_data']['ResponseMetricName']==metric]
+    effectiveness=selected_metric.groupby(by=['MediaChannelName'])['ResponseMetricValue'].sum()
+    effectiveness_df=pd.DataFrame({'Channel':effectiveness.index,"ResponseMetricValue":effectiveness.values})
+
+    summary_df_sorted=summary_df_sorted.merge(effectiveness_df,left_on="Channel_name",right_on='Channel')
+
+    # st.dataframe(summary_df_sorted.head(2))
+    summary_df_sorted['Efficiency']=summary_df_sorted['ResponseMetricValue']/summary_df_sorted['Optimized_spend']
+# # # st.dataframe(summary_df_sorted.head(2))
+# st.dataframe(summary_df_sorted.head(2))
+
+    columns= st.columns(3)
+    with columns[0]:
+        fig=summary_plot(summary_df_sorted, x='Optimized_spend', y='Channel_name', title='', text_column='Optimized_spend',color='Channel_name')
+        st.plotly_chart(fig,use_container_width=True)  
+    with columns[1]: 
+        
+        # effectiveness=(selected_metric.groupby(by=['MediaChannelName'])['ResponseMetricValue'].sum()).values
+        # effectiveness_df=pd.DataFrame({'Channel':st.session_state['raw_data']['MediaChannelName'].unique(),"ResponseMetricValue":effectiveness})
+        # # effectiveness.reset_index(inplace=True)
+        # # st.dataframe(effectiveness.head())
+        fig=summary_plot(summary_df_sorted, x='ResponseMetricValue', y='Channel_name', title='Effectiveness', text_column='ResponseMetricValue',color='Channel_name')
+        st.plotly_chart(fig,use_container_width=True)  
+
+    with columns[2]:
+        fig=summary_plot(summary_df_sorted, x='Efficiency', y='Channel_name', title='Efficiency', text_column='Efficiency',color='Channel_name',format_as_decimal=True)
+        st.plotly_chart(fig,use_container_width=True)
+
+import plotly.express as px
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+
+# Create figure with subplots
+# fig = make_subplots(rows=1, cols=2)
+
+# # Add funnel plot to subplot 1
+# fig.add_trace(
+#     go.Funnel(
+#         x=effectiveness_overall[~(effectiveness_overall['ResponseMetricName'].isin(['Total Approved Accounts - Revenue', 'BAU approved clients - Revenue', 'Gamified approved clients - Revenue', "Total Approved Accounts - Appsflyer"]))]['ResponseMetricValue'],
+#         y=effectiveness_overall[~(effectiveness_overall['ResponseMetricName'].isin(['Total Approved Accounts - Revenue', 'BAU approved clients - Revenue', 'Gamified approved clients - Revenue', "Total Approved Accounts - Appsflyer"]))]['ResponseMetricName'],
+#         textposition="inside",
+#         texttemplate="%{x:.2s}",
+#         customdata=effectiveness_overall['Efficiency'],
+#         hovertemplate="%{customdata:.2f}<extra></extra>"
+#     ),
+#     row=1, col=1
+# )
+
+# # Add bar plot to subplot 2 
+# fig.add_trace(
+#     go.Bar(
+#         x=effectiveness_overall.sort_values(by='ResponseMetricValue')['Efficiency'], 
+#         y=effectiveness_overall.sort_values(by='ResponseMetricValue')['ResponseMetricName'],
+#         marker_color=effectiveness_overall['ResponseMetric'], 
+#         customdata=effectiveness_overall['Efficiency'],
+#         hovertemplate="%{customdata:.2f}<extra></extra>",
+#         textposition="outside"
+#     ),
+#     row=1, col=2
+# )
+
+# # Update layout
+# fig.update_layout(title_text="Effectiveness")
+# fig.update_yaxes(title_text="", row=1, col=1)
+# fig.update_yaxes(title_text="", showticklabels=False, row=1, col=2) 
+# fig.update_xaxes(title_text="Efficiency", showticklabels=False, row=1, col=2)
+
+# # Show figure
+# st.plotly_chart(fig)

pages/1_Data_Validation.py

@@ -0,0 +1,241 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from Eda_functions import *
+import numpy as np
+import re
+import pickle
+from ydata_profiling import ProfileReport
+from streamlit_pandas_profiling import st_profile_report
+import streamlit as st
+import streamlit.components.v1 as components
+import sweetviz as sv
+from utilities import set_header,initialize_data,load_local_css
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
+from st_aggrid import GridOptionsBuilder
+from st_aggrid import AgGrid
+import base64
+
+st.set_page_config(
+  page_title="Data Validation",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+load_local_css('styles.css')
+set_header()
+
+
+
+#preprocessing
+# with open('Categorised_data.pkl', 'rb') as file:
+#   Categorised_data = pickle.load(file)
+# with open("edited_dataframe.pkl", 'rb') as file:
+
+
+# df = pickle.load(file)
+# date=df.index
+# df.reset_index(inplace=True)
+# df['Date'] = pd.to_datetime(date)
+
+
+#prospects=pd.read_excel('EDA_Data.xlsx',sheet_name='Prospects')
+#spends=pd.read_excel('EDA_Data.xlsx',sheet_name='SPEND INPUT')
+#spends.columns=['Week','Streaming (Spends)','TV (Spends)','Search (Spends)','Digital (Spends)']
+#df=pd.concat([df,spends],axis=1)
+
+#df['Date'] =pd.to_datetime(df['Date']).dt.strftime('%m/%d/%Y')
+#df['Prospects']=prospects['Prospects']
+#df.drop(['Week'],axis=1,inplace=True)
+
+
+st.title('Data Validation and Insights')
+
+with open("Pickle_files/main_df",'rb') as f:
+   st.session_state['cleaned_data']= pickle.load(f)
+with open("Pickle_files/category_dict",'rb') as c:
+   st.session_state['category_dict']=pickle.load(c)
+
+# st.write(st.session_state['cleaned_data'])
+    
+target_variables=[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Response_Metric']
+
+
+target_column = st.selectbox('Select the Target Feature/Dependent Variable (will be used in all charts as reference)',list(*target_variables))
+st.session_state['target_column']=target_column
+
+
+fig=line_plot_target(st.session_state['cleaned_data'], target=target_column, title=f'{target_column} Over Time')
+st.plotly_chart(fig, use_container_width=True)
+
+
+media_channel=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Media'])
+# st.write(media_channel)
+
+Non_media_channel=[col for col in st.session_state['cleaned_data'].columns if col not in media_channel]
+
+
+st.markdown('### Annual Data Summary')
+st.dataframe(summary(st.session_state['cleaned_data'], media_channel+[target_column], spends=None,Target=True), use_container_width=True)
+
+if st.checkbox('Show raw data'):
+    st.write(pd.concat([pd.to_datetime(st.session_state['cleaned_data']['Date']).dt.strftime('%m/%d/%Y'),st.session_state['cleaned_data'].select_dtypes(np.number).applymap(format_numbers)],axis=1))
+col1 = st.columns(1)
+
+if "selected_feature" not in st.session_state:
+    st.session_state['selected_feature']=None
+
+st.header('1. Media Channels')
+
+if 'Validation' not in st.session_state:
+    st.session_state['Validation']=[]
+
+eda_columns=st.columns(2)
+with eda_columns[0]:
+    if st.button('Generate Profile Report'):
+        pr = st.session_state['cleaned_data'].profile_report()
+
+        pr.to_file("Profile_Report.html")
+
+        with open("Profile_Report.html", "rb") as f:
+            profile_report_html = f.read()
+        b64 = base64.b64encode(profile_report_html).decode()
+        href = f'<a href="data:text/html;base64,{b64}" download="Profile_Report.html">Download Profile Report</a>'
+        st.markdown(href, unsafe_allow_html=True)
+
+with eda_columns[1]:
+    if st.button('Generate Sweetviz Report'):
+    
+        def generate_report_with_target(df, target_feature):
+            report = sv.analyze([df, "Dataset"], target_feat=target_feature)
+            return report
+
+        report = generate_report_with_target(st.session_state['cleaned_data'], target_feature=target_column)
+        report.show_html()
+
+
+selected_media = st.selectbox('Select media', np.unique([Categorised_data[col]['VB'] for col in media_channel]))
+# selected_feature=st.multiselect('Select Metric', df.columns[df.columns.str.contains(selected_media,case=False)])
+st.session_state["selected_feature"]=st.selectbox('Select Metric',[col for col in  media_channel  if    Categorised_data[col]['VB'] in selected_media ] )
+spends_features=[col for col in df.columns if 'spends' in col.lower() or 'cost' in col.lower()]
+spends_feature=[col for col in spends_features if col.split('_')[0] in st.session_state["selected_feature"].split('_')[0]]
+#st.write(spends_features)
+#st.write(spends_feature)
+#st.write(selected_feature)
+
+
+val_variables=[col for col in media_channel if col!='Date']
+if len(spends_feature)==0:  
+    st.warning('No spends varaible available for the selected metric in data') 
+       
+else:
+    st.write(f'Selected spends variable {spends_feature[0]} if wrong please name the varaibles properly')
+    # Create the dual-axis line plot
+    fig_row1 = line_plot(df, x_col='Date', y1_cols=[st.session_state["selected_feature"]], y2_cols=[target_column], title=f'Analysis of {st.session_state["selected_feature"]} and {[target_column][0]} Over Time')
+    st.plotly_chart(fig_row1, use_container_width=True)
+    st.markdown('### Annual Data Summary')
+    st.dataframe(summary(df,[st.session_state["selected_feature"]],spends=spends_feature[0]),use_container_width=True)
+    if st.button('Validate'):
+        st.session_state['Validation'].append(st.session_state["selected_feature"])
+
+    if st.checkbox('Validate all'):
+        st.session_state['Validation'].extend(val_variables)
+        st.success('All media variables are validated ✅')
+    if len(set(st.session_state['Validation']).intersection(val_variables))!=len(val_variables):
+        #st.write(st.session_state['Validation'])
+        validation_data=pd.DataFrame({'Variables':val_variables,
+                                    'Validated':[1 if col in st.session_state['Validation'] else 0 for col in val_variables],
+                                    'Bucket':[Categorised_data[col]['VB'] for col in val_variables]})
+        gd=GridOptionsBuilder.from_dataframe(validation_data)
+        gd.configure_pagination(enabled=True)
+        gd.configure_selection(use_checkbox=True,selection_mode='multiple')
+        #gd.configure_selection_toggle_all(None, show_toggle_all=True)
+        #gd.configure_columns_auto_size_mode(GridOptionsBuilder.configure_columns)
+        gridoptions=gd.build()
+        #st.text(st.session_state['Validation'])
+        table = AgGrid(validation_data,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED,fit_columns_on_grid_load=True)
+        #st.table(table)
+        selected_rows = table["selected_rows"]
+        st.session_state['Validation'].extend([col['Variables'] for col in selected_rows])
+        not_validated_variables = [col for col in val_variables if col not in st.session_state["Validation"]]
+        if not_validated_variables:
+            not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}'
+            st.warning(not_validated_message)
+
+
+
+st.header('2. Non Media Variables')
+selected_columns_row = [col for col in df.columns if ("imp" not in col.lower()) and ('cli' not in col.lower() ) and ('spend' not in col.lower()) and col!='Date']
+selected_columns_row4 = st.selectbox('Select Channel',selected_columns_row )
+if not selected_columns_row4: 
+    st.warning('Please select at least one.')
+else:
+    # Create the dual-axis line plot
+    fig_row4 = line_plot(df, x_col='Date', y1_cols=[selected_columns_row4], y2_cols=[target_column], title=f'Analysis of {selected_columns_row4} and {target_column} Over Time')
+    st.plotly_chart(fig_row4, use_container_width=True)
+    selected_non_media=selected_columns_row4
+    sum_df = df[['Date', selected_non_media,target_column]]
+    sum_df['Year']=pd.to_datetime(df['Date']).dt.year
+    #st.dataframe(df)
+    #st.dataframe(sum_df.head(2))
+    sum_df=sum_df.groupby('Year').agg('sum')
+    sum_df.loc['Grand Total']=sum_df.sum()         
+    sum_df=sum_df.applymap(format_numbers) 
+    sum_df.fillna('-',inplace=True)
+    sum_df=sum_df.replace({"0.0":'-','nan':'-'})
+    st.markdown('### Annual Data Summary')    
+    st.dataframe(sum_df,use_container_width=True)
+
+    # if st.checkbox('Validate',key='2'):
+    #     st.session_state['Validation'].append(selected_columns_row4)
+# val_variables=[col for col in media_channel if col!='Date']
+# if st.checkbox('Validate all'):
+#     st.session_state['Validation'].extend(val_variables)
+# validation_data=pd.DataFrame({'Variables':val_variables,
+#                             'Validated':[1 if col in st.session_state['Validation'] else 0 for col in val_variables],
+#                             'Bucket':[Categorised_data[col]['VB'] for col in val_variables]})
+# gd=GridOptionsBuilder.from_dataframe(validation_data)
+# gd.configure_pagination(enabled=True)
+# gd.configure_selection(use_checkbox=True,selection_mode='multiple')
+# #gd.configure_selection_toggle_all(None, show_toggle_all=True)
+# #gd.configure_columns_auto_size_mode(GridOptionsBuilder.configure_columns)
+# gridoptions=gd.build()
+# #st.text(st.session_state['Validation'])
+# table = AgGrid(validation_data,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED,fit_columns_on_grid_load=True)
+# #st.table(table)
+# selected_rows = table["selected_rows"]
+# st.session_state['Validation'].extend([col['Variables'] for col in selected_rows])
+# not_validated_variables = [col for col in val_variables if col not in st.session_state["Validation"]]
+# if not_validated_variables:
+#     not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}'
+#     st.warning(not_validated_message)
+
+options = list(df.select_dtypes(np.number).columns)
+st.markdown(' ')
+st.markdown(' ')
+st.markdown('# Exploratory Data Analysis')
+st.markdown(' ')
+
+selected_options = []
+num_columns = 4
+num_rows = -(-len(options) // num_columns)  # Ceiling division to calculate rows
+
+# Create a grid of checkboxes
+st.header('Select Features for Correlation Plot')
+tick=False
+if st.checkbox('Select all'):
+    tick=True
+selected_options = []
+for row in range(num_rows):
+    cols = st.columns(num_columns)
+    for col in cols:
+        if options:
+            option = options.pop(0) 
+            selected = col.checkbox(option,value=tick)
+            if selected:
+                selected_options.append(option)
+# Display selected options
+#st.write('You selected:', selected_options)
+st.pyplot(correlation_plot(df,selected_options,target_column))
+

pages/2_Transformations.py

@@ -0,0 +1,612 @@
+'''
+MMO Build Sprint 3
+date :
+additions : adding more variables to session state for saved model : random effect, predicted train & test
+'''
+
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from Eda_functions import format_numbers
+import numpy as np
+import pickle
+from st_aggrid import AgGrid
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
+from utilities import set_header,load_local_css
+from st_aggrid import GridOptionsBuilder
+import time
+import itertools
+import statsmodels.api as sm
+import numpy as npc
+import re
+import itertools
+from sklearn.metrics import mean_absolute_error, r2_score,mean_absolute_percentage_error
+from sklearn.preprocessing import MinMaxScaler
+import os
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+import statsmodels.api as sm
+import statsmodels.formula.api as smf
+
+from datetime import datetime
+import seaborn as sns
+from Data_prep_functions import *
+
+
+def get_random_effects(media_data, panel_col, mdf):
+    random_eff_df = pd.DataFrame(columns=[panel_col, "random_effect"])
+
+    for i, market in enumerate(media_data[panel_col].unique()):
+        print(i, end='\r')
+        intercept = mdf.random_effects[market].values[0]
+        random_eff_df.loc[i, 'random_effect'] = intercept
+        random_eff_df.loc[i, panel_col] = market
+
+    return random_eff_df
+
+
+def mdf_predict(X_df, mdf, random_eff_df) :
+    X=X_df.copy()
+    X['fixed_effect'] = mdf.predict(X)
+    X=pd.merge(X, random_eff_df, on=panel_col, how='left')
+    X['pred'] = X['fixed_effect'] + X['random_effect']
+    # X.to_csv('Test/megred_df.csv',index=False)
+    X.drop(columns=['fixed_effect', 'random_effect'], inplace=True)
+    return X['pred']
+
+st.set_page_config(
+  page_title="Model Build",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+
+load_local_css('styles.css')
+set_header()
+
+
+st.title('1. Build Your Model')
+
+# set the panel column
+date_col = 'date'
+
+
+media_data=pd.read_csv(r'upf_data_converted.csv')
+# with open("Pickle_files/main_df",'rb') as f:
+#    media_data= pickle.load(f)
+   
+
+media_data.columns=[i.lower().strip().replace(' ','_').replace('-','').replace(':','').replace("__", "_") for i in media_data.columns]
+#st.write(media_data.columns)
+#media_data.drop(['indicacao_impressions','infleux_impressions','influencer_impressions'],axis=1,inplace=True)
+target_col = 'total_approved_accounts_revenue'
+# st.write(media_data.columns)
+media_data.sort_values(date_col, inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+
+date=media_data[date_col]
+st.session_state['date']=date
+revenue=media_data[target_col]
+media_data.drop([target_col],axis=1,inplace=True)
+media_data.drop([date_col],axis=1,inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+
+
+if st.toggle('Apply Transformations on DMA/Panel Level'):
+  dma=st.selectbox('Select the Level of data ',[ col for col in media_data.columns if col.lower() in ['dma','panel', 'markets']])
+  panel_col= dma
+
+else:
+  #""" code to aggregate data on date """
+
+
+  dma=None
+
+# dma_dict={ dm:media_data[media_data[dma]==dm] for dm in media_data[dma].unique()}
+# st.write(dma_dict)
+
+st.markdown('## Select the Range of Transformations')
+columns = st.columns(2)
+old_shape=media_data.shape
+
+
+if "old_shape" not in st.session_state:
+   st.session_state['old_shape']=old_shape
+
+
+with columns[0]:
+  slider_value_adstock  = st.slider('Select Adstock Range (only applied to media)', 0.0, 1.0, (0.2, 0.4), step=0.1, format="%.2f")
+with columns[1]:
+  slider_value_lag = st.slider('Select Lag Range (applied to media, seasonal, macroeconomic variables)', 1, 7, (1, 3), step=1)
+
+# with columns[2]:
+#    slider_value_power=st.slider('Select Power range (only applied to media )',0,4,(1,2),step=1)
+
+# with columns[1]:
+#    st.number_input('Select the range of half saturation point ',min_value=1,max_value=5)
+#    st.number_input('Select the range of  ')
+
+# Section 1 - Transformations Functions
+def lag(data,features,lags,dma=None):
+    if dma:
+
+        transformed_data=pd.concat([data.groupby([dma])[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+        return pd.concat([transformed_data,data],axis=1)
+
+    else:
+
+        #''' data should be aggregated on date'''
+
+        transformed_data=pd.concat([data[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+
+        return pd.concat([transformed_data,data],axis=1)
+
+#adstock
+def adstock(df, alphas, cutoff, features,dma=None):
+    # st.write(features)
+
+    if dma:
+        transformed_data=pd.DataFrame()
+        for d in df[dma].unique():
+            dma_sub_df = df[df[dma] == d]
+            n = len(dma_sub_df)
+
+
+            weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+            X = dma_sub_df[features].to_numpy()
+
+            res = pd.DataFrame(np.hstack(weights @ X),
+                               columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+
+            transformed_data=pd.concat([transformed_data,res],axis=0)
+            transformed_data.reset_index(drop=True,inplace=True)
+        return pd.concat([transformed_data,df],axis=1)
+
+    else:
+
+        n = len(df)
+
+
+        weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+
+        X = df[features].to_numpy()
+        res = pd.DataFrame(np.hstack(weights @ X),
+                           columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+        return  pd.concat([res,df],axis=1)
+
+
+
+
+# Section 2 - Begin Transformations
+
+if 'media_data' not in st.session_state:
+
+  st.session_state['media_data']=pd.DataFrame()
+
+# Sprint3 additions
+if 'random_effects' not in st.session_state:
+  st.session_state['random_effects']=pd.DataFrame()
+if 'pred_train' not in st.session_state:
+  st.session_state['pred_train'] = []
+if 'pred_test' not in st.session_state:
+  st.session_state['pred_test'] = []
+# end of Sprint3 additions
+
+# variables_to_be_transformed=[col for col in media_data.columns if col.lower() not in ['dma','panel'] ] # change for buckets
+variables_to_be_transformed=[col for col in media_data.columns if '_clicks' in col.lower() or '_impress' in col.lower()] # srishti - change
+# st.write(variables_to_be_transformed)
+# st.write(media_data[variables_to_be_transformed].dtypes)
+
+with columns[0]:
+  if st.button('Apply Transformations'):
+    with st.spinner('Applying Transformations'):
+      transformed_data_lag=lag(media_data,features=variables_to_be_transformed,lags=np.arange(slider_value_lag[0],slider_value_lag[1]+1,1),dma=dma)
+
+      # variables_to_be_transformed=[col for col in list(transformed_data_lag.columns) if col not in ['Date','DMA','Panel']] #change for buckets
+      variables_to_be_transformed = [col for col in media_data.columns if
+                                    '_clicks' in col.lower() or '_impress' in col.lower()]  # srishti - change
+
+      transformed_data_adstock=adstock(df=transformed_data_lag, alphas=np.arange(slider_value_adstock[0],slider_value_adstock[1],0.1), cutoff=8, features=variables_to_be_transformed,dma=dma)
+
+      # st.success('Done')
+      st.success("Transformations complete!")
+
+      st.write(f'old shape {old_shape}, new shape {transformed_data_adstock.shape}')
+      # st.write(media_data.head(10))
+      # st.write(transformed_data_adstock.head(10))
+
+      transformed_data_adstock.columns = [c.replace(".","_") for c in transformed_data_adstock.columns] # srishti
+      # st.write(transformed_data_adstock.columns)
+      st.session_state['media_data']=transformed_data_adstock # srishti
+
+    # with st.spinner('Applying Transformations'):
+    #   time.sleep(2)
+    #   st.success("Transformations complete!")
+
+# if st.session_state['media_data'].shape[1]>old_shape[1]:
+  # with columns[0]:
+    # st.write(f'Total no.of variables before transformation: {old_shape[1]}, Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+  #st.write(f'Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+
+# Section 3 - Create combinations
+
+# bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','FB: Level Achieved - Tier 1 Impressions',
+#       ' FB: Level Achieved - Tier 2 Impressions','paid_social_others',
+#         ' GA App: Will And Cid Pequena Baixo Risco Clicks',
+#       'digital_tactic_others',"programmatic"
+#       ]
+
+# srishti - bucket names changed
+bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','fb_level_achieved_tier_2',
+      'fb_level_achieved_tier_1','paid_social_others',
+        'ga_app',
+      'digital_tactic_others',"programmatic"
+      ]
+
+with columns[1]:
+  if st.button('Create Combinations of Variables'):
+
+    top_3_correlated_features=[]
+    # for col in st.session_state['media_data'].columns[:19]:
+    original_cols = [c for c in st.session_state['media_data'].columns if "_clicks" in c.lower() or "_impressions" in c.lower()]
+    original_cols = [c for c in original_cols if "_lag" not in c.lower() and "_adstock" not in c.lower()]
+    # st.write(original_cols)
+
+    # for col in st.session_state['media_data'].columns[:19]:
+    for col in original_cols: # srishti - new
+        corr_df=pd.concat([st.session_state['media_data'].filter(regex=col),
+                  revenue],axis=1).corr()[target_col].iloc[:-1]
+        top_3_correlated_features.append(list(corr_df.sort_values(ascending=False).head(2).index))
+        # st.write(col, top_3_correlated_features)
+    flattened_list = [item for sublist in top_3_correlated_features for item in sublist]
+    # all_features_set={var:[col for col in flattened_list if var in col] for var in bucket}
+    all_features_set={var:[col for col in flattened_list if var in col] for var in bucket if len([col for col in flattened_list if var in col])>0} # srishti
+
+    channels_all=[values for values in all_features_set.values()]
+    # st.write(channels_all)
+    st.session_state['combinations'] = list(itertools.product(*channels_all))
+  # if 'combinations' not in st.session_state:
+  #   st.session_state['combinations']=combinations_all
+
+    st.session_state['final_selection']=st.session_state['combinations']
+    st.success('Done')
+    # st.write(f"{len(st.session_state['combinations'])} combinations created")
+
+
+    revenue.reset_index(drop=True,inplace=True)
+  if 'Model_results' not in st.session_state:
+        st.session_state['Model_results']={'Model_object':[],
+      'Model_iteration':[],
+      'Feature_set':[],
+      'MAPE':[],
+      'R2':[],
+      'ADJR2':[]
+      }
+
+  def reset_model_result_dct():
+      st.session_state['Model_results']={'Model_object':[],
+      'Model_iteration':[],
+      'Feature_set':[],
+      'MAPE':[],
+      'R2':[],
+      'ADJR2':[]
+      }
+
+      # if st.button('Build Model'):
+  if 'iterations' not in st.session_state:
+    st.session_state['iterations']=0
+      # st.write("1",st.session_state["final_selection"])
+
+  if 'final_selection' not in st.session_state:
+      st.session_state['final_selection']=False
+
+save_path = r"Model/"
+with columns[1]:
+  if  st.session_state['final_selection']:
+    st.write(f'Total combinations created {format_numbers(len(st.session_state["final_selection"]))}')
+
+    
+if st.checkbox('Build all iterations'):
+   iterations=len(st.session_state['final_selection'])
+else:
+   iterations = st.number_input('Select the number of iterations to perform', min_value=0, step=100, value=st.session_state['iterations'],on_change=reset_model_result_dct)
+  #  st.write("iterations=", iterations)
+
+if st.button('Build Model',on_click=reset_model_result_dct):
+  st.session_state['iterations']=iterations 
+  # st.write("2",st.session_state["final_selection"])
+
+  # Section 4 - Model
+
+  st.session_state['media_data']=st.session_state['media_data'].fillna(method='ffill')
+  st.markdown(
+      'Data Split -- Training Period: May 9th, 2023 - October 5th,2023 , Testing Period: October 6th, 2023 - November 7th, 2023 ')
+  progress_bar = st.progress(0)  # Initialize the progress bar
+  # time_remaining_text = st.empty()  # Create an empty space for time remaining text
+  start_time = time.time()  # Record the start time
+  progress_text = st.empty()
+  # time_elapsed_text = st.empty()
+  # for i, selected_features in enumerate(st.session_state["final_selection"][40000:40000 + int(iterations)]):
+  # st.write(st.session_state["final_selection"])
+  # for i, selected_features in enumerate(st.session_state["final_selection"]):
+  for i, selected_features in enumerate(st.session_state["final_selection"][0:int(iterations)]): # srishti
+      df = st.session_state['media_data']
+
+      fet = [var for var in selected_features if len(var) > 0]
+      inp_vars_str = " + ".join(fet)  # new
+      
+
+      X = df[fet]
+      y = revenue
+      ss = MinMaxScaler()
+      X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+      # X = sm.add_constant(X)
+
+      X['total_approved_accounts_revenue'] = revenue  # Sprint2
+      X[panel_col] = df[panel_col] # Sprint2
+
+
+
+      X_train=X.iloc[:8000]
+      X_test=X.iloc[8000:]
+      y_train=y.iloc[:8000]
+      y_test=y.iloc[8000:]
+
+
+
+      md = smf.mixedlm("total_approved_accounts_revenue ~ {}".format(inp_vars_str),
+                      data=X_train[['total_approved_accounts_revenue'] + fet],
+                      groups=X_train[panel_col])
+      mdf = md.fit()
+      predicted_values = mdf.fittedvalues
+
+      # st.write(fet)
+      # positive_coeff=fet
+      # negetive_coeff=[]
+
+      coefficients = mdf.fe_params.to_dict()
+      model_possitive = [col for col in coefficients.keys() if coefficients[col] > 0]
+      # st.write(positive_coeff)
+      # st.write(model_possitive)
+      pvalues = [var for var in list(mdf.pvalues) if var <= 0.06]
+
+      # if (len(model_possitive) / len(selected_features)) > 0.9 and (len(pvalues) / len(selected_features)) >= 0.8:
+      if (len(model_possitive) / len(selected_features)) > 0 and (len(pvalues) / len(selected_features)) >= 0: # srishti - changed just for testing, revert later
+          # predicted_values = model.predict(X_train)
+          mape = mean_absolute_percentage_error(y_train, predicted_values)
+          r2 = r2_score(y_train, predicted_values)
+          adjr2 = 1 - (1 - r2) * (len(y_train) - 1) / (len(y_train) - len(selected_features) - 1)
+
+          filename = os.path.join(save_path, f"model_{i}.pkl")
+          with open(filename, "wb") as f:
+              pickle.dump(mdf, f)
+          # with open(r"C:\Users\ManojP\Documents\MMM\simopt\Model\model.pkl", 'rb') as file:
+          #   model = pickle.load(file)
+
+          st.session_state['Model_results']['Model_object'].append(filename)
+          st.session_state['Model_results']['Model_iteration'].append(i)
+          st.session_state['Model_results']['Feature_set'].append(fet)
+          st.session_state['Model_results']['MAPE'].append(mape)
+          st.session_state['Model_results']['R2'].append(r2)
+          st.session_state['Model_results']['ADJR2'].append(adjr2)
+
+      current_time = time.time()
+      time_taken = current_time - start_time
+      time_elapsed_minutes = time_taken / 60
+      completed_iterations_text = f"{i + 1}/{iterations}"
+      progress_bar.progress((i + 1) / int(iterations))
+      progress_text.text(f'Completed iterations: {completed_iterations_text},Time Elapsed (min): {time_elapsed_minutes:.2f}')
+
+  st.write(f'Out of {st.session_state["iterations"]} iterations : {len(st.session_state["Model_results"]["Model_object"])} valid models')
+  pd.DataFrame(st.session_state['Model_results']).to_csv('model_output.csv')
+
+  def to_percentage(value):
+    return f'{value * 100:.1f}%'
+
+## Section 5 - Select Model
+st.title('2. Select Models')
+if 'tick' not in st.session_state:
+   st.session_state['tick']=False
+if st.checkbox('Show results of top 10 models (based on MAPE and Adj. R2)',value=st.session_state['tick']):
+  st.session_state['tick']=True
+  st.write('Select one model iteration to generate performance metrics for it:')
+  data=pd.DataFrame(st.session_state['Model_results'])
+  data.sort_values(by=['MAPE'],ascending=False,inplace=True)
+  data.drop_duplicates(subset='Model_iteration',inplace=True)
+  top_10=data.head(10)
+  top_10['Rank']=np.arange(1,len(top_10)+1,1)
+  top_10[['MAPE','R2','ADJR2']]=np.round(top_10[['MAPE','R2','ADJR2']],4).applymap(to_percentage)
+  top_10_table = top_10[['Rank','Model_iteration','MAPE','ADJR2','R2']]
+  #top_10_table.columns=[['Rank','Model Iteration Index','MAPE','Adjusted R2','R2']]
+  gd=GridOptionsBuilder.from_dataframe(top_10_table)
+  gd.configure_pagination(enabled=True)
+  gd.configure_selection(use_checkbox=True)
+
+
+  gridoptions=gd.build()
+
+  table = AgGrid(top_10,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED)
+
+  selected_rows=table.selected_rows
+  # if st.session_state["selected_rows"] != selected_rows:
+  #   st.session_state["build_rc_cb"] = False
+  st.session_state["selected_rows"] = selected_rows
+  if 'Model' not in st.session_state:
+    st.session_state['Model']={}
+
+# Section 6 - Display Results
+
+  if len(selected_rows)>0:
+    st.header('2.1 Results Summary')
+
+    model_object=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Model_object']
+    features_set=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Feature_set']
+
+    with open(str(model_object.values[0]), 'rb') as file:
+        # print(file)
+        model = pickle.load(file)
+    st.write(model.summary())
+    st.header('2.2 Actual vs. Predicted Plot')
+
+    df=st.session_state['media_data']
+    X=df[features_set.values[0]]
+    # X = sm.add_constant(X)
+    y=revenue
+
+    ss = MinMaxScaler()
+    X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+
+    # Sprint2 changes
+    X['total_approved_accounts_revenue'] = revenue  # new
+    X[panel_col] = df[panel_col]
+    X[date_col]=date
+
+
+
+    X_train=X.iloc[:8000]
+    X_test=X.iloc[8000:].reset_index(drop=True)
+    y_train=y.iloc[:8000]
+    y_test=y.iloc[8000:].reset_index(drop=True)
+
+
+    random_eff_df = get_random_effects(media_data, panel_col, model)
+    train_pred = model.fittedvalues
+    test_pred = mdf_predict(X_test, model, random_eff_df)
+    print("__"*20, test_pred.isna().sum())
+
+    # save x test to test - srishti
+    x_test_to_save = X_test.copy()
+    x_test_to_save['Actuals'] = y_test
+    x_test_to_save['Predictions'] = test_pred
+
+    x_train_to_save=X_train.copy()
+    x_train_to_save['Actuals'] = y_train
+    x_train_to_save['Predictions'] = train_pred
+
+    x_train_to_save.to_csv('Test/x_train_to_save.csv',index=False)
+    x_test_to_save.to_csv('Test/x_test_to_save.csv',index=False)
+
+    st.session_state['X']=X_train
+    st.session_state['features_set']=features_set.values[0]
+    print("**"*20,"selected model features : ",features_set.values[0])
+    metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(X_train[date_col], y_train, train_pred, model,target_column='Revenue',is_panel=True) # Sprint2
+
+    st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+
+
+    st.markdown('## 2.3 Residual Analysis')
+    columns=st.columns(2)
+    with columns[0]:
+      fig=plot_residual_predicted(y_train,train_pred,X_train) # Sprint2
+      st.plotly_chart(fig)
+
+    with columns[1]:
+      st.empty()
+      fig = qqplot(y_train,train_pred) # Sprint2
+      st.plotly_chart(fig)
+
+    with columns[0]:
+      fig=residual_distribution(y_train,train_pred) # Sprint2
+      st.pyplot(fig)
+
+
+
+    vif_data = pd.DataFrame()
+    # X=X.drop('const',axis=1)
+    X_train_with_panels = X_train.copy() # Sprint2 -- creating a copy of xtrain. Later deleting panel, target & date from xtrain
+    X_train.drop(columns=[target_col, panel_col, date_col], inplace=True) # Sprint2
+    vif_data["Variable"] = X_train.columns
+    vif_data["VIF"] = [variance_inflation_factor(X_train.values, i) for i in range(X_train.shape[1])]
+    vif_data.sort_values(by=['VIF'],ascending=False,inplace=True)
+    vif_data=np.round(vif_data)
+    vif_data['VIF']=vif_data['VIF'].astype(float)
+    st.header('2.4 Variance Inflation Factor (VIF)')
+    #st.dataframe(vif_data)
+    color_mapping = {
+    'darkgreen': (vif_data['VIF'] < 3),
+    'orange': (vif_data['VIF'] >= 3) & (vif_data['VIF'] <= 10),
+    'darkred': (vif_data['VIF'] > 10)
+    }
+
+# Create a horizontal bar plot
+    fig, ax = plt.subplots()
+    fig.set_figwidth(10)  # Adjust the width of the figure as needed
+
+    # Sort the bars by descending VIF values
+    vif_data = vif_data.sort_values(by='VIF', ascending=False)
+
+    # Iterate through the color mapping and plot bars with corresponding colors
+    for color, condition in color_mapping.items():
+        subset = vif_data[condition]
+        bars = ax.barh(subset["Variable"], subset["VIF"], color=color, label=color)
+
+        # Add text annotations on top of the bars
+        for bar in bars:
+            width = bar.get_width()
+            ax.annotate(f'{width:}', xy=(width, bar.get_y() + bar.get_height() / 2), xytext=(5, 0),
+                        textcoords='offset points', va='center')
+
+    # Customize the plot
+    ax.set_xlabel('VIF Values')
+    #ax.set_title('2.4 Variance Inflation Factor (VIF)')
+    #ax.legend(loc='upper right')
+
+    # Display the plot in Streamlit
+    st.pyplot(fig)
+
+
+
+    with st.expander('Results Summary Test data'):
+      # ss = MinMaxScaler()
+      # X_test = pd.DataFrame(ss.fit_transform(X_test), columns=X_test.columns)
+      st.header('2.2 Actual vs. Predicted Plot')
+
+      metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(X_test[date_col], y_test, test_pred, model,target_column='Revenue',is_panel=True) # Sprint2
+
+      st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+      st.markdown('## 2.3 Residual Analysis')
+      columns=st.columns(2)
+      with columns[0]:
+        fig=plot_residual_predicted(revenue,test_pred,X_test) # Sprint2
+        st.plotly_chart(fig)
+
+      with columns[1]:
+        st.empty()
+        fig = qqplot(revenue,test_pred) # Sprint2
+        st.plotly_chart(fig)
+
+      with columns[0]:
+        fig=residual_distribution(revenue,test_pred) # Sprint2
+        st.pyplot(fig)
+
+    value=False
+    if st.checkbox('Save this model to tune',key='build_rc_cb'):
+      mod_name=st.text_input('Enter model name')
+      if len(mod_name)>0:
+        st.session_state['Model'][mod_name]={"Model_object":model,'feature_set':st.session_state['features_set'],'X_train':X_train_with_panels}
+        st.session_state['X_train']=X_train_with_panels
+        st.session_state['X_test']=X_test
+        st.session_state['y_train']=y_train
+        st.session_state['y_test']=y_test
+
+        # Sprint3 additions
+        random_eff_df= get_random_effects(media_data, panel_col, model)
+        st.session_state['random_effects']=random_eff_df
+
+        st.session_state['pred_train']=model.fittedvalues
+        st.session_state['pred_test']=mdf_predict(X_test, model, random_eff_df)
+        # End of Sprint3 additions
+
+        with open("best_models.pkl", "wb") as f:
+          pickle.dump(st.session_state['Model'], f)
+          st.success('Model saved! Proceed to the next page to tune the model')
+        value=False
+
+        # st.write(st.session_state['Model'][mod_name]['X_train'].columns)
+        # st.write(st.session_state['X_test'].columns)

pages/3_Model_Tuning.py

@@ -0,0 +1,437 @@
+'''
+MMO Build Sprint 3
+date :
+changes : capability to tune MixedLM as well as simple LR in the same page
+'''
+
+import streamlit as st
+import pandas as pd
+from Eda_functions import format_numbers
+import pickle
+from utilities import set_header,load_local_css
+import statsmodels.api as sm
+import re
+from sklearn.preprocessing import MinMaxScaler
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+import statsmodels.formula.api as smf
+from Data_prep_functions import *
+
+for i in ["model_tuned", "X_train_tuned", "X_test_tuned", "tuned_model_features"] :
+    if i not in st.session_state :
+        st.session_state[i] = None
+
+st.set_page_config(
+  page_title="Model Tuning",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+load_local_css('styles.css')
+set_header()
+
+# Sprint3
+is_panel= True
+panel_col= 'dma' # set the panel column
+date_col = 'date'
+target_col = 'total_approved_accounts_revenue'
+
+st.title('1. Model Tuning')
+
+
+if "X_train" not in st.session_state:
+   st.error(
+"Oops! It seems there are no saved models available. Please build and save a model from the previous page to proceed.")
+   st.stop()
+X_train=st.session_state['X_train']
+X_test=st.session_state['X_test']
+y_train=st.session_state['y_train']
+y_test=st.session_state['y_test']
+df=st.session_state['media_data']
+
+# st.write(X_train.columns)
+# st.write(X_test.columns)
+
+with open("best_models.pkl", 'rb') as file:
+  model_dict= pickle.load(file)
+
+if 'selected_model' not in st.session_state:
+   st.session_state['selected_model']=0
+
+# st.write(model_dict[st.session_state["selected_model"]]['X_train'].columns)
+
+st.markdown('### 1.1 Event Flags')
+st.markdown('Helps in quantifying the impact of specific occurrences of events')
+with st.expander('Apply Event Flags'):
+  st.session_state["selected_model"]=st.selectbox('Select Model to apply flags',model_dict.keys())
+  model =model_dict[st.session_state["selected_model"]]['Model_object']
+  date=st.session_state['date']
+  date=pd.to_datetime(date)
+  X_train =model_dict[st.session_state["selected_model"]]['X_train']
+
+  features_set= model_dict[st.session_state["selected_model"]]['feature_set']
+
+  col=st.columns(3)
+  min_date=min(date)
+  max_date=max(date)
+  with col[0]:
+    start_date=st.date_input('Select Start Date',min_date,min_value=min_date,max_value=max_date)
+  with col[1]:
+    end_date=st.date_input('Select End Date',max_date,min_value=min_date,max_value=max_date)
+  with col[2]:
+    repeat=st.selectbox('Repeat Annually',['Yes','No'],index=1)
+  if repeat =='Yes':
+      repeat=True
+  else:
+      repeat=False
+  # X_train=sm.add_constant(X_train)
+
+  if 'Flags' not in st.session_state:
+    st.session_state['Flags']={}
+  # print("**"*50)
+  # print(y_train)
+  # print("**"*50)
+  # print(model.fittedvalues)
+  if is_panel : # Sprint3
+      met, line_values, fig_flag = plot_actual_vs_predicted(X_train[date_col], y_train,
+                                                            model.fittedvalues, model,
+                                                            target_column='Revenue',
+                                                            flag=(start_date, end_date),
+                                                            repeat_all_years=repeat, is_panel=True)
+      st.plotly_chart(fig_flag, use_container_width=True)
+
+      # create flag on test
+      met, test_line_values, fig_flag = plot_actual_vs_predicted(X_test[date_col], y_test,
+                                                            st.session_state['pred_test'], model,
+                                                            target_column='Revenue',
+                                                            flag=(start_date, end_date),
+                                                            repeat_all_years=repeat, is_panel=True)
+
+  else :
+      met,line_values,fig_flag=plot_actual_vs_predicted(date[:150], y_train, model.predict(X_train), model,flag=(start_date,end_date),repeat_all_years=repeat)
+      st.plotly_chart(fig_flag,use_container_width=True)
+
+      met,test_line_values,fig_flag=plot_actual_vs_predicted(date[150:], y_test, model.predict(X_test), model,flag=(start_date,end_date),repeat_all_years=repeat)
+
+
+  flag_name='f1'
+  flag_name=st.text_input('Enter Flag Name')
+  if st.button('Update flag'):
+    st.session_state['Flags'][flag_name]= {}
+    st.session_state['Flags'][flag_name]['train']=line_values
+    st.session_state['Flags'][flag_name]['test']=test_line_values
+    # st.write(st.session_state['Flags'][flag_name])
+    st.success(f'{flag_name} stored')
+
+  options=list(st.session_state['Flags'].keys())
+  selected_options = []
+  num_columns = 4
+  num_rows = -(-len(options) // num_columns)
+
+
+tick=False
+if st.checkbox('Select all'):
+    tick=True
+selected_options = []
+for row in range(num_rows):
+    cols = st.columns(num_columns)
+    for col in cols:
+        if options:
+            option = options.pop(0)
+            selected = col.checkbox(option,value=tick)
+            if selected:
+                selected_options.append(option)
+
+st.markdown('### 1.2 Select Parameters to Apply')
+parameters=st.columns(3)
+with parameters[0]:
+   Trend=st.checkbox("**Trend**")
+   st.markdown('Helps account for long-term trends or seasonality that could influence advertising effectiveness')
+with parameters[1]:
+   week_number=st.checkbox('**Week_number**')
+   st.markdown('Assists in detecting and incorporating weekly patterns or seasonality')
+with parameters[2]:
+   sine_cosine=st.checkbox('**Sine and Cosine Waves**')
+   st.markdown('Helps in capturing cyclical patterns or seasonality in the data')
+
+if st.button('Build model with Selected Parameters and Flags'):
+  st.header('2.1 Results Summary')
+  # date=list(df.index)
+  # df = df.reset_index(drop=True)
+  # st.write(df.head(2))
+  # X_train=df[features_set]
+  ss = MinMaxScaler()
+  if is_panel == True :
+    X = X_train[features_set]
+    X_train_tuned = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+    X_train_tuned[target_col] = X_train[target_col]
+    X_train_tuned[date_col] = X_train[date_col]
+    X_train_tuned[panel_col] = X_train[panel_col]
+
+    X = X_test[features_set]
+    X_test_tuned = pd.DataFrame(ss.transform(X), columns=X.columns)
+    X_test_tuned[target_col] = X_test[target_col]
+    X_test_tuned[date_col] = X_test[date_col]
+    X_test_tuned[panel_col] = X_test[panel_col]
+
+  else :
+    X_train_tuned = pd.DataFrame(ss.fit_transform(X_train), columns=X_train.columns)
+    X_train_tuned = sm.add_constant(X_train_tuned)
+
+    X_test_tuned = pd.DataFrame(ss.transform(X_test), columns=X_test.columns)
+    X_test_tuned = sm.add_constant(X_test_tuned)
+
+  for flag in selected_options:
+    X_train_tuned[flag]=st.session_state['Flags'][flag]['train']
+    X_test_tuned[flag]=st.session_state['Flags'][flag]['test']
+
+    #test
+    # X_train_tuned.to_csv("Test/X_train_tuned_flag.csv",index=False)
+    # X_test_tuned.to_csv("Test/X_test_tuned_flag.csv",index=False)
+
+  new_features = features_set
+    # print("()()"*20,flag, len(st.session_state['Flags'][flag]))
+  if Trend:
+     # Sprint3 - group by panel, calculate trend of each panel spearately. Add trend to new feature set
+     if is_panel :
+         newdata = pd.DataFrame()
+         panel_wise_end_point_train = {}
+         for panel, groupdf in X_train_tuned.groupby(panel_col):
+             groupdf.sort_values(date_col, inplace=True)
+             groupdf['Trend'] = np.arange(1, len(groupdf) + 1, 1)
+             newdata = pd.concat([newdata, groupdf])
+             panel_wise_end_point_train[panel] = len(groupdf)
+         X_train_tuned = newdata.copy()
+
+         test_newdata=pd.DataFrame()
+         for panel, test_groupdf in X_test_tuned.groupby(panel_col):
+             test_groupdf.sort_values(date_col, inplace=True)
+             start = panel_wise_end_point_train[panel]+1
+             end = start + len(test_groupdf)
+             # print("??"*20, panel, len(test_groupdf), len(np.arange(start, end, 1)), start)
+             test_groupdf['Trend'] = np.arange(start, end, 1)
+             test_newdata = pd.concat([test_newdata, test_groupdf])
+         X_test_tuned = test_newdata.copy()
+
+         new_features = new_features + ['Trend']
+
+         # test
+         X_test_tuned.to_csv("Test/X_test_tuned_trend.csv", index=False)
+         X_train_tuned.to_csv("Test/X_train_tuned_trend.csv", index=False)
+         pd.concat([X_train_tuned,X_test_tuned]).sort_values([panel_col, date_col]).to_csv("Test/X_train_test_tuned_trend.csv", index=False)
+
+     else :
+         X_train_tuned['Trend']=np.arange(1,len(X_train_tuned)+1,1)
+         X_test_tuned['Trend'] = np.arange(len(X_train_tuned)+1, len(X_train_tuned)+len(X_test_tuned), 1)
+
+  if week_number :
+     # Sprint3 - create weeknumber from date column in xtrain tuned. add week num to new feature set
+     if is_panel :
+        X_train_tuned[date_col] = pd.to_datetime(X_train_tuned[date_col])
+        X_train_tuned['Week_number'] = X_train_tuned[date_col].dt.day_of_week
+        if X_train_tuned['Week_number'].nunique() == 1 :
+            st.write("All dates in the data are of the same week day. Hence Week number can't be used.")
+        else :
+            X_test_tuned[date_col] = pd.to_datetime(X_test_tuned[date_col])
+            X_test_tuned['Week_number'] = X_test_tuned[date_col].dt.day_of_week
+            new_features = new_features + ['Week_number']
+
+     else :
+        date = pd.to_datetime(date.values)
+        X_train_tuned['Week_number'] = date.dt.day_of_week[:150]
+        X_test_tuned['Week_number'] = date.dt.day_of_week[150:]
+
+  if sine_cosine :
+      # Sprint3 - create panel wise sine cosine waves in xtrain tuned. add to new feature set
+      if is_panel :
+        new_features = new_features + ['sine_wave', 'cosine_wave']
+        newdata = pd.DataFrame()
+        groups = X_train_tuned.groupby(panel_col)
+        frequency = 2 * np.pi / 365  # Adjust the frequency as needed
+
+        train_panel_wise_end_point = {}
+        for panel, groupdf in groups:
+            num_samples = len(groupdf)
+            train_panel_wise_end_point[panel] = num_samples
+            days_since_start = np.arange(num_samples)
+            sine_wave = np.sin(frequency * days_since_start)
+            cosine_wave = np.cos(frequency * days_since_start)
+            sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+            assert len(sine_cosine_df) == len(groupdf)
+            # groupdf = pd.concat([groupdf, sine_cosine_df], axis=1)
+            groupdf['sine_wave'] = sine_wave
+            groupdf['cosine_wave'] = cosine_wave
+            newdata = pd.concat([newdata, groupdf])
+
+        test_groups = X_test_tuned.groupby(panel_col)
+        for panel, test_groupdf in test_groups:
+            num_samples = len(test_groupdf)
+            start = train_panel_wise_end_point[panel]
+            days_since_start = np.arange(start, start+num_samples, 1)
+            # print("##", panel, num_samples, start, len(np.arange(start, start+num_samples, 1)))
+            sine_wave = np.sin(frequency * days_since_start)
+            cosine_wave = np.cos(frequency * days_since_start)
+            sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+            assert len(sine_cosine_df) == len(test_groupdf)
+            # groupdf = pd.concat([groupdf, sine_cosine_df], axis=1)
+            test_groupdf['sine_wave'] = sine_wave
+            test_groupdf['cosine_wave'] = cosine_wave
+            newdata = pd.concat([newdata, test_groupdf])
+
+        X_train_tuned = newdata.copy()
+
+
+      else :
+        num_samples = len(X_train_tuned)
+        frequency = 2 * np.pi / 365  # Adjust the frequency as needed
+        days_since_start = np.arange(num_samples)
+        sine_wave = np.sin(frequency * days_since_start)
+        cosine_wave = np.cos(frequency * days_since_start)
+        sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+        # Concatenate the sine and cosine waves with the scaled X DataFrame
+        X_train_tuned = pd.concat([X_train_tuned, sine_cosine_df], axis=1)
+
+        test_num_samples = len(X_test_tuned)
+        start = num_samples
+        days_since_start = np.arange(start, start+test_num_samples, 1)
+        sine_wave = np.sin(frequency * days_since_start)
+        cosine_wave = np.cos(frequency * days_since_start)
+        sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+        # Concatenate the sine and cosine waves with the scaled X DataFrame
+        X_test_tuned = pd.concat([X_test_tuned, sine_cosine_df], axis=1)
+
+  # model
+
+  if is_panel :
+      if selected_options :
+        new_features =  new_features + selected_options
+
+      inp_vars_str = " + ".join(new_features)
+
+      # X_train_tuned.to_csv("Test/X_train_tuned.csv",index=False)
+      # st.write(X_train_tuned[['total_approved_accounts_revenue'] + new_features].dtypes)
+      # st.write(X_train_tuned[['total_approved_accounts_revenue', panel_col] + new_features].isna().sum())
+
+      md_tuned = smf.mixedlm("total_approved_accounts_revenue ~ {}".format(inp_vars_str),
+                             data=X_train_tuned[['total_approved_accounts_revenue'] + new_features],
+                             groups=X_train_tuned[panel_col])
+      model_tuned = md_tuned.fit()
+
+
+
+      # plot act v pred for original model and tuned model
+      metrics_table, line, actual_vs_predicted_plot = plot_actual_vs_predicted(X_train[date_col], y_train,
+                                                                               model.fittedvalues, model,
+                                                                               target_column='Revenue',
+                                                                               is_panel=True)
+      metrics_table_tuned, line, actual_vs_predicted_plot_tuned = plot_actual_vs_predicted(X_train_tuned[date_col],
+                                                                                           X_train_tuned[target_col],
+                                                                                           model_tuned.fittedvalues,
+                                                                                           model_tuned,
+                                                                                           target_column='Revenue',
+                                                                                           is_panel=True)
+
+  else :
+      model_tuned = sm.OLS(y_train, X_train_tuned).fit()
+
+      metrics_table, line, actual_vs_predicted_plot = plot_actual_vs_predicted(date[:150], y_train,
+                                                                               model.predict(X_train), model,
+                                                                               target_column='Revenue')
+      metrics_table_tuned, line, actual_vs_predicted_plot_tuned = plot_actual_vs_predicted(date[:150], y_train,
+                                                                                           model_tuned.predict(
+                                                                                               X_train_tuned),
+                                                                                           model_tuned,
+                                                                                           target_column='Revenue')
+
+  # st.write(metrics_table_tuned)
+  mape=np.round(metrics_table.iloc[0,1],2)
+  r2=np.round(metrics_table.iloc[1,1],2)
+  adjr2=np.round(metrics_table.iloc[2,1],2)
+
+  mape_tuned=np.round(metrics_table_tuned.iloc[0,1],2)
+  r2_tuned=np.round(metrics_table_tuned.iloc[1,1],2)
+  adjr2_tuned=np.round(metrics_table_tuned.iloc[2,1],2)
+
+  parameters_=st.columns(3)
+  with parameters_[0]:
+     st.metric('R2',r2_tuned,np.round(r2_tuned-r2,2))
+  with parameters_[1]:
+     st.metric('Adjusted R2',adjr2_tuned,np.round(adjr2_tuned-adjr2,2))
+  with parameters_[2]:
+     st.metric('MAPE',mape_tuned,np.round(mape_tuned-mape,2),'inverse')
+
+  st.header('2.2 Actual vs. Predicted Plot')
+  # if is_panel:
+  #   metrics_table, line, actual_vs_predicted_plot = plot_actual_vs_predicted(date, y_train, model.predict(X_train),
+  #                                                                              model, target_column='Revenue',is_panel=True)
+  # else:
+  #   metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_train, model.predict(X_train), model,target_column='Revenue')
+
+  metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(X_train_tuned[date_col], X_train_tuned[target_col],
+                                                                                  model_tuned.fittedvalues, model_tuned,
+                                                                                  target_column='Revenue',
+                                                                                  is_panel=True)
+      # plot_actual_vs_predicted(X_train[date_col], y_train,
+      #                                                                             model.fittedvalues, model,
+      #                                                                             target_column='Revenue',
+      #                                                                             is_panel=is_panel)
+
+  st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+
+
+  st.markdown('## 2.3 Residual Analysis')
+  columns=st.columns(2)
+  with columns[0]:
+    fig=plot_residual_predicted(y_train,model.predict(X_train),X_train)
+    st.plotly_chart(fig)
+
+  with columns[1]:
+    st.empty()
+    fig = qqplot(y_train,model.predict(X_train))
+    st.plotly_chart(fig)
+
+  with columns[0]:
+    fig=residual_distribution(y_train,model.predict(X_train))
+    st.pyplot(fig)
+
+  if st.checkbox('Use this model to build response curves',key='123'):
+    st.session_state["tuned_model"] = model_tuned
+    st.session_state["X_train_tuned"] = X_train_tuned
+    st.session_state["X_test_tuned"] = X_test_tuned
+    st.session_state["X_train_tuned"] = X_train_tuned
+    st.session_state["X_test_tuned"] = X_test_tuned
+    if is_panel :
+        st.session_state["tuned_model_features"] = new_features
+    with open("tuned_model.pkl", "wb") as f:
+        pickle.dump(st.session_state['tuned_model'], f)
+        st.success('Model saved!')
+
+#   raw_data=df[features_set]
+#   columns_raw=[re.split(r"(_lag|_adst)",col)[0] for col in raw_data.columns]
+#   raw_data.columns=columns_raw
+#   columns_media=[col for col in columns_raw if Categorised_data[col]['BB']=='Media']
+#   raw_data=raw_data[columns_media]
+
+#   raw_data['Date']=list(df.index)
+
+#   spends_var=[col for col in df.columns if "spends" in col.lower() and 'adst' not in col.lower() and 'lag' not in col.lower()]
+#   spends_df=df[spends_var]
+#   spends_df['Week']=list(df.index)
+
+
+#   j=0
+#   X1=X.copy()
+#   col=X1.columns
+#   for i in model.params.values:
+#       X1[col[j]]=X1.iloc[:,j]*i
+#       j+=1
+#   contribution_df=X1
+#   contribution_df['Date']=list(df.index)
+#   excel_file='Overview_data.xlsx'
+
+#   with pd.ExcelWriter(excel_file,engine='xlsxwriter') as writer:
+#      raw_data.to_excel(writer,sheet_name='RAW DATA MMM',index=False)
+#      spends_df.to_excel(writer,sheet_name='SPEND INPUT',index=False)
+#      contribution_df.to_excel(writer,sheet_name='CONTRIBUTION MMM')

pages/4_Saved_Model_Results.py

@@ -0,0 +1,413 @@
+import plotly.express as px
+import numpy as np
+import plotly.graph_objects as go
+import streamlit as st
+import pandas as pd
+import statsmodels.api as sm
+from sklearn.metrics import mean_absolute_percentage_error
+import sys
+import os
+from utilities import (set_header, 
+                       load_local_css,
+                       load_authenticator)
+import seaborn as sns
+import matplotlib.pyplot as plt
+import sweetviz as sv
+import tempfile
+from sklearn.preprocessing import MinMaxScaler
+from st_aggrid import AgGrid
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
+from st_aggrid import GridOptionsBuilder
+import sys
+import re
+
+sys.setrecursionlimit(10**6)
+
+original_stdout = sys.stdout
+sys.stdout = open('temp_stdout.txt', 'w')
+sys.stdout.close()
+sys.stdout = original_stdout
+
+st.set_page_config(layout='wide')
+load_local_css('styles.css')
+set_header()
+
+for k, v in st.session_state.items():
+    if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+        st.session_state[k] = v
+
+authenticator = st.session_state.get('authenticator')
+if authenticator is None:
+    authenticator = load_authenticator()
+
+name, authentication_status, username = authenticator.login('Login', 'main')
+auth_status = st.session_state.get('authentication_status')
+
+if auth_status == True:
+    is_state_initiaized = st.session_state.get('initialized',False)
+    if not is_state_initiaized:
+        a=1
+      
+
+    def plot_residual_predicted(actual, predicted, df_):
+            df_['Residuals'] = actual - pd.Series(predicted)
+            df_['StdResidual'] = (df_['Residuals'] - df_['Residuals'].mean()) / df_['Residuals'].std()
+            
+            # Create a Plotly scatter plot
+            fig = px.scatter(df_, x=predicted, y='StdResidual', opacity=0.5,color_discrete_sequence=["#11B6BD"])
+            
+            # Add horizontal lines
+            fig.add_hline(y=0, line_dash="dash", line_color="darkorange")
+            fig.add_hline(y=2, line_color="red")
+            fig.add_hline(y=-2, line_color="red")
+            
+            fig.update_xaxes(title='Predicted')
+            fig.update_yaxes(title='Standardized Residuals (Actual - Predicted)')
+            
+            # Set the same width and height for both figures
+            fig.update_layout(title='Residuals over Predicted Values', autosize=False, width=600, height=400)
+            
+            return fig
+
+    def residual_distribution(actual, predicted):
+            Residuals = actual - pd.Series(predicted)
+            
+            # Create a Seaborn distribution plot
+            sns.set(style="whitegrid")
+            plt.figure(figsize=(6, 4))
+            sns.histplot(Residuals, kde=True, color="#11B6BD")
+            
+            plt.title(' Distribution of Residuals')
+            plt.xlabel('Residuals')
+            plt.ylabel('Probability Density')
+            
+            return plt
+
+    
+    def qqplot(actual, predicted):
+            Residuals = actual - pd.Series(predicted)
+            Residuals = pd.Series(Residuals)
+            Resud_std = (Residuals - Residuals.mean()) / Residuals.std()
+            
+            # Create a QQ plot using Plotly with custom colors
+            fig = go.Figure()
+            fig.add_trace(go.Scatter(x=sm.ProbPlot(Resud_std).theoretical_quantiles,
+                                    y=sm.ProbPlot(Resud_std).sample_quantiles,
+                                    mode='markers',
+                                    marker=dict(size=5, color="#11B6BD"),
+                                    name='QQ Plot'))
+            
+            # Add the 45-degree reference line
+            diagonal_line = go.Scatter(
+                x=[-2, 2],  # Adjust the x values as needed to fit the range of your data
+                y=[-2, 2],  # Adjust the y values accordingly
+                mode='lines',
+                line=dict(color='red'),  # Customize the line color and style
+                name=' '
+            )
+            fig.add_trace(diagonal_line)
+            
+            # Customize the layout
+            fig.update_layout(title='QQ Plot of Residuals',title_x=0.5, autosize=False, width=600, height=400,
+                            xaxis_title='Theoretical Quantiles', yaxis_title='Sample Quantiles')
+            
+            return fig
+
+
+    def plot_actual_vs_predicted(date, y, predicted_values, model):
+
+        fig = go.Figure()
+
+        fig.add_trace(go.Scatter(x=date, y=y, mode='lines', name='Actual', line=dict(color='blue')))
+        fig.add_trace(go.Scatter(x=date, y=predicted_values, mode='lines', name='Predicted', line=dict(color='orange')))
+        
+        # Calculate MAPE
+        mape = mean_absolute_percentage_error(y, predicted_values)*100
+        
+        # Calculate R-squared
+        rss = np.sum((y - predicted_values) ** 2)
+        tss = np.sum((y - np.mean(y)) ** 2)
+        r_squared = 1 - (rss / tss)
+        
+        # Get the number of predictors
+        num_predictors = model.df_model
+        
+        # Get the number of samples
+        num_samples = len(y)
+        
+        # Calculate Adjusted R-squared
+        adj_r_squared = 1 - ((1 - r_squared) * ((num_samples - 1) / (num_samples - num_predictors - 1)))
+        metrics_table = pd.DataFrame({
+        'Metric': ['MAPE', 'R-squared', 'AdjR-squared'],
+        'Value': [mape, r_squared, adj_r_squared]})
+        fig.update_layout(
+            xaxis=dict(title='Date'),
+            yaxis=dict(title='Value'),
+            title=f'MAPE : {mape:.2f}%, AdjR2: {adj_r_squared:.2f}',
+            xaxis_tickangle=-30
+        )
+
+        return metrics_table,fig
+    def contributions(X, model):
+        X1 = X.copy()
+        for j, col in enumerate(X1.columns):
+            X1[col] = X1[col] * model.params.values[j]
+
+        return np.round((X1.sum() / sum(X1.sum()) * 100).sort_values(ascending=False), 2)
+
+    transformed_data=pd.read_csv('transformed_data.csv')
+
+    # hard coded for now, need to get features set from model
+
+    feature_set_dct={'app_installs_-_appsflyer':['paid_search_clicks',
+                                            'fb:_level_achieved_-_tier_1_impressions_lag2',
+                                            'fb:_level_achieved_-_tier_2_clicks_lag2',
+                                            'paid_social_others_impressions_adst.1',
+                                            'ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag2',
+                                            'digital_tactic_others_clicks',
+                                            'kwai_clicks_adst.3',
+                                            'programmaticclicks',
+                                            'indicacao_clicks_adst.1',
+                                            'infleux_clicks_adst.4',
+                                            'influencer_clicks'],
+                                            
+                'account_requests_-_appsflyer':['paid_search_impressions',
+                                                'fb:_level_achieved_-_tier_1_clicks_adst.1',
+                                                'fb:_level_achieved_-_tier_2_clicks_adst.1',
+                                                'paid_social_others_clicks_lag2',
+                                                'ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag5_adst.1',
+                                                'digital_tactic_others_clicks_adst.1',
+                                                'kwai_clicks_adst.2',
+                                                'programmaticimpressions_lag4_adst.1',
+                                                'indicacao_clicks',
+                                                'infleux_clicks_adst.2',
+                                                'influencer_clicks'],
+
+                'total_approved_accounts_-_appsflyer':['paid_search_clicks',
+                                                        'fb:_level_achieved_-_tier_1_impressions_lag2_adst.1',
+                                                        'fb:_level_achieved_-_tier_2_impressions_lag2',
+                                                        'paid_social_others_clicks_lag2_adst.2',
+                                                        'ga_app:_will_and_cid_pequena_baixo_risco_impressions_lag4',
+                                                        'digital_tactic_others_clicks',
+                                                        'kwai_impressions_adst.2',
+                                                        'programmaticclicks_adst.5',
+                                                        'indicacao_clicks_adst.1',
+                                                        'infleux_clicks_adst.3',
+                                                        'influencer_clicks'],
+
+                'total_approved_accounts_-_revenue':['paid_search_impressions_adst.5',
+                                                    'kwai_impressions_lag2_adst.3',
+                                                    'indicacao_clicks_adst.3',
+                                                    'infleux_clicks_adst.3',
+                                                    'programmaticclicks_adst.4',
+                                                    'influencer_clicks_adst.3',
+                                                    'fb:_level_achieved_-_tier_1_impressions_adst.2',
+                                                    'fb:_level_achieved_-_tier_2_impressions_lag3_adst.5',
+                                                    'paid_social_others_impressions_adst.3',
+                                                    'ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag3_adst.5',
+                                                    'digital_tactic_others_clicks_adst.2']                       
+                                            
+                }
+
+    #""" the above part should be modified so that we are fetching features set from the saved model"""
+
+
+
+    def contributions(X, model,target):
+        X1 = X.copy()
+        for j, col in enumerate(X1.columns):
+            X1[col] = X1[col] * model.params.values[j]
+        
+        contributions= np.round((X1.sum() / sum(X1.sum()) * 100).sort_values(ascending=False), 2)
+        contributions=pd.DataFrame(contributions,columns=target).reset_index().rename(columns={'index':'Channel'})
+        contributions['Channel']=[ re.split(r'_imp|_cli', col)[0] for col in contributions['Channel']]
+        
+        return contributions
+    
+
+    def model_fit(features_set,target):
+        X = transformed_data[features_set]
+        y=  transformed_data[target]
+        ss = MinMaxScaler()
+        X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+        X = sm.add_constant(X)
+        X_train=X.iloc[:150]
+        X_test=X.iloc[150:]
+        y_train=y.iloc[:150]
+        y_test=y.iloc[150:]
+        model = sm.OLS(y_train, X_train).fit()
+        predicted_values_train = model.predict(X_train)
+        r2 = model.rsquared
+        adjr2 = model.rsquared_adj
+        train_mape = mean_absolute_percentage_error(y_train, predicted_values_train)
+        test_mape=mean_absolute_percentage_error(y_test, model.predict(X_test))
+        summary=model.summary()
+        train_contributions=contributions(X_train,model,[target])
+        return pd.DataFrame({'Model':target,'R2':np.round(r2,2),'ADJr2':np.round(adjr2,2),'Train Mape':np.round(train_mape,2),
+                             'Test Mape':np.round(test_mape,2),'Summary':summary,'Model_object':model
+                             },index=[0]), train_contributions
+
+    metrics_table=pd.DataFrame()
+
+    if 'contribution_df' not in st.session_state:
+        st.session_state["contribution_df"]=pd.DataFrame()
+
+    for target,feature_set in feature_set_dct.items():
+       metrics_table= pd.concat([metrics_table,model_fit(features_set=feature_set,target=target)[0]])
+       if st.session_state["contribution_df"].empty:
+           st.session_state["contribution_df"]= model_fit(features_set=feature_set,target=target)[1]
+       else:
+        st.session_state["contribution_df"]=pd.merge(st.session_state["contribution_df"],model_fit(features_set=feature_set,target=target)[1])
+
+    # st.write(st.session_state["contribution_df"])
+        
+
+    metrics_table.reset_index(drop=True,inplace=True)
+
+
+
+    
+    
+
+
+
+    eda_columns=st.columns(2)
+    with eda_columns[1]:
+        eda=st.button('Generate EDA Report',help="Click to generate a bivariate report for the selected response metric from the table below.")
+    
+
+
+    # st.markdown('Model Metrics')
+        
+    st.title('Contribution Overview')
+
+    contribution_selections=st.multiselect('Select the models to compare contributions',[col for col in st.session_state['contribution_df'].columns if col.lower() != 'channel'   ],default=[col for col in st.session_state['contribution_df'].columns if col.lower() != 'channel'   ][-1])
+    trace_data=[]
+
+    for selection in contribution_selections:
+
+        trace=go.Bar(x=st.session_state['contribution_df']['Channel'], y=st.session_state['contribution_df'][selection],name=selection,text=np.round(st.session_state['contribution_df'][selection],0).astype(int).astype(str)+'%',textposition='outside')
+        trace_data.append(trace)
+
+    layout = go.Layout(
+    title='Metrics Contribution by Channel',
+    xaxis=dict(title='Channel Name'),
+    yaxis=dict(title='Metrics Contribution'),
+    barmode='group'
+                )
+    fig = go.Figure(data=trace_data, layout=layout)
+    st.plotly_chart(fig,use_container_width=True)
+
+    st.title('Analysis of Models Result')
+    #st.markdown()
+    gd_table=metrics_table.iloc[:,:-2]
+    gd=GridOptionsBuilder.from_dataframe(gd_table)
+    #gd.configure_pagination(enabled=True)
+    gd.configure_selection(use_checkbox=True)
+
+
+    gridoptions=gd.build()
+    table = AgGrid(gd_table,gridOptions=gridoptions,fit_columns_on_grid_load=True,height=200)
+    # table=metrics_table.iloc[:,:-2]
+    # table.insert(0, "Select", False)
+    # selection_table=st.data_editor(table,column_config={"Select": st.column_config.CheckboxColumn(required=True)})
+    
+
+
+    if len(table.selected_rows)==0:
+        st.warning("Click on the checkbox to view comprehensive results of the selected model.")
+        st.stop()
+    else:    
+        target_column=table.selected_rows[0]['Model']
+        feature_set=feature_set_dct[target_column]
+
+    with eda_columns[1]:
+        if eda:
+            def generate_report_with_target(channel_data, target_feature):
+                report = sv.analyze([channel_data, "Dataset"], target_feat=target_feature,verbose=False)
+                temp_dir = tempfile.mkdtemp()
+                report_path = os.path.join(temp_dir, "report.html")
+                report.show_html(filepath=report_path, open_browser=False)  # Generate the report as an HTML file
+                return report_path
+            
+            report_data=transformed_data[feature_set]
+            report_data[target_column]=transformed_data[target_column]
+            report_file = generate_report_with_target(report_data, target_column)
+        
+            if os.path.exists(report_file):
+                with open(report_file, 'rb') as f:
+                    st.download_button(
+                        label="Download EDA Report",
+                        data=f.read(),
+                        file_name="report.html",
+                        mime="text/html"
+                    )
+            else:
+                st.warning("Report generation failed. Unable to find the report file.")
+
+
+
+    model=metrics_table[metrics_table['Model']==target_column]['Model_object'].iloc[0]
+    st.header('Model Summary')
+    st.write(model.summary())
+    X=transformed_data[feature_set]
+    ss=MinMaxScaler()
+    X=pd.DataFrame(ss.fit_transform(X),columns=X.columns)
+    X=sm.add_constant(X)
+    y=transformed_data[target_column]
+    X_train=X.iloc[:150]
+    X_test=X.iloc[150:]
+    y_train=y.iloc[:150]
+    y_test=y.iloc[150:]
+    X.index=transformed_data['date']
+    y.index=transformed_data['date']
+
+    metrics_table_train,fig_train= plot_actual_vs_predicted(X_train.index, y_train, model.predict(X_train), model)
+    metrics_table_test,fig_test= plot_actual_vs_predicted(X_test.index, y_test, model.predict(X_test), model)
+
+    metrics_table_train=metrics_table_train.set_index('Metric').transpose()
+    metrics_table_train.index=['Train']
+    metrics_table_test=metrics_table_test.set_index('Metric').transpose()
+    metrics_table_test.index=['test']
+    metrics_table=np.round(pd.concat([metrics_table_train,metrics_table_test]),2) 
+
+    st.markdown('Result Overview')
+    st.dataframe(np.round(metrics_table,2),use_container_width=True)
+
+    st.subheader('Actual vs Predicted Plot Train')
+
+    st.plotly_chart(fig_train,use_container_width=True)
+    st.subheader('Actual vs Predicted Plot Test')
+    st.plotly_chart(fig_test,use_container_width=True)
+
+    st.markdown('## Residual Analysis')
+    columns=st.columns(2)
+
+
+    Xtrain1=X_train.copy()
+    with columns[0]:
+        fig=plot_residual_predicted(y_train,model.predict(Xtrain1),Xtrain1)
+        st.plotly_chart(fig)
+
+    with columns[1]:
+        st.empty()
+        fig = qqplot(y_train,model.predict(X_train))
+        st.plotly_chart(fig)
+
+    with columns[0]:
+        fig=residual_distribution(y_train,model.predict(X_train))
+        st.pyplot(fig)
+
+
+
+elif auth_status == False:
+    st.error('Username/Password is incorrect')
+    try:
+        username_forgot_pw, email_forgot_password, random_password = authenticator.forgot_password('Forgot password')
+        if username_forgot_pw:
+            st.success('New password sent securely')
+            # Random password to be transferred to the user securely
+        elif username_forgot_pw == False:
+            st.error('Username not found')
+    except Exception as e:
+        st.error(e)

pages/5_Model_Result_Overview.py

@@ -0,0 +1,103 @@
+import streamlit as st
+from utilities import (set_header,
+                       initialize_data,
+                       load_local_css,
+                       create_channel_summary,
+                       create_contribution_pie,
+                       create_contribuion_stacked_plot,
+                       create_channel_spends_sales_plot,
+                       format_numbers,
+                       channel_name_formating,
+                       load_authenticator)
+import plotly.graph_objects as go
+import streamlit_authenticator as stauth
+import yaml
+from yaml import SafeLoader
+import time
+
+st.set_page_config(layout='wide')
+load_local_css('styles.css')
+set_header()
+
+target='Revenue'
+# for k, v in st.session_state.items():
+
+#     if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+#         st.session_state[k] = v
+
+# authenticator = st.session_state.get('authenticator')
+
+# if authenticator is None:
+#     authenticator = load_authenticator()
+    
+# name, authentication_status, username = authenticator.login('Login', 'main')
+# auth_status = st.session_state['authentication_status']
+
+# if auth_status:
+#     authenticator.logout('Logout', 'main')
+    
+#     is_state_initiaized = st.session_state.get('initialized',False)
+#     if not is_state_initiaized:
+initialize_data()
+scenario = st.session_state['scenario']
+raw_df = st.session_state['raw_df']
+st.header('Overview of previous spends')
+
+
+columns = st.columns((1,1,3))
+
+with columns[0]:
+    st.metric(label = 'Spends', value=format_numbers(float(scenario.actual_total_spends)))
+###print(f"##################### {scenario.actual_total_sales} ##################")
+with columns[1]:
+    st.metric(label = target, value=format_numbers(float(scenario.actual_total_sales),include_indicator=False))
+
+
+actual_summary_df = create_channel_summary(scenario)
+actual_summary_df['Channel'] = actual_summary_df['Channel'].apply(channel_name_formating) 
+
+columns = st.columns((2,1))
+with columns[0]:
+    with st.expander('Channel wise overview'):
+        st.markdown(actual_summary_df.style.set_table_styles(
+        [{
+            'selector': 'th',
+            'props': [('background-color', '#11B6BD')]
+        },
+            {
+            'selector' : 'tr:nth-child(even)',
+            'props' : [('background-color', '#11B6BD')]
+            }]).to_html(), unsafe_allow_html=True)
+        
+st.markdown("<hr>",unsafe_allow_html=True)
+##############################
+
+st.plotly_chart(create_contribution_pie(),use_container_width=True)
+st.markdown("<hr>",unsafe_allow_html=True)
+
+
+################################3
+st.plotly_chart(create_contribuion_stacked_plot(scenario),use_container_width=True)
+st.markdown("<hr>",unsafe_allow_html=True)
+#######################################
+
+selected_channel_name = st.selectbox('Channel', st.session_state['channels_list'] + ['non media'], format_func=channel_name_formating)
+selected_channel = scenario.channels.get(selected_channel_name,None)
+
+st.plotly_chart(create_channel_spends_sales_plot(selected_channel), use_container_width=True)
+
+st.markdown("<hr>",unsafe_allow_html=True)
+
+# elif auth_status == False:
+#     st.error('Username/Password is incorrect')
+    
+# if auth_status != True:
+#     try:
+#         username_forgot_pw, email_forgot_password, random_password = authenticator.forgot_password('Forgot password')
+#         if username_forgot_pw:
+#             st.success('New password sent securely')
+#             # Random password to be transferred to user securely
+#         elif username_forgot_pw == False:
+#             st.error('Username not found')
+#     except Exception as e:
+#         st.error(e)

pages/6_Build_Response_Curves.py

@@ -0,0 +1,168 @@
+import streamlit as st
+import plotly.express as px
+import numpy as np
+import plotly.graph_objects as go
+from utilities import channel_name_formating, load_authenticator, initialize_data
+from sklearn.metrics import r2_score
+from collections import OrderedDict
+from classes import class_from_dict,class_to_dict
+import pickle
+import json
+
+for k, v in st.session_state.items():
+    if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+        st.session_state[k] = v
+
+def s_curve(x,K,b,a,x0):
+    return K / (1 + b*np.exp(-a*(x-x0)))
+
+def save_scenario(scenario_name):
+    """
+    Save the current scenario with the mentioned name in the session state
+    
+    Parameters
+    ----------
+    scenario_name
+        Name of the scenario to be saved
+    """
+    if 'saved_scenarios' not in st.session_state:
+        st.session_state = OrderedDict()
+        
+    #st.session_state['saved_scenarios'][scenario_name] = st.session_state['scenario'].save()
+    st.session_state['saved_scenarios'][scenario_name] = class_to_dict(st.session_state['scenario'])
+    st.session_state['scenario_input'] = ""
+    print(type(st.session_state['saved_scenarios']))
+    with open('../saved_scenarios.pkl', 'wb') as f:
+        pickle.dump(st.session_state['saved_scenarios'],f)
+
+
+def reset_curve_parameters():
+    del st.session_state['K']
+    del st.session_state['b']
+    del st.session_state['a']
+    del st.session_state['x0']
+    
+def update_response_curve():
+    # st.session_state['rcs'][selected_channel_name]['K'] = st.session_state['K']
+    # st.session_state['rcs'][selected_channel_name]['b'] = st.session_state['b']
+    # st.session_state['rcs'][selected_channel_name]['a'] = st.session_state['a']
+    # st.session_state['rcs'][selected_channel_name]['x0'] = st.session_state['x0']
+    # rcs = st.session_state['rcs']
+    _channel_class = st.session_state['scenario'].channels[selected_channel_name]
+    _channel_class.update_response_curves({
+                           'K'  : st.session_state['K'], 
+                           'b'  : st.session_state['b'], 
+                           'a'  : st.session_state['a'],
+                           'x0' : st.session_state['x0']})
+    
+
+# authenticator = st.session_state.get('authenticator')
+# if authenticator is None:
+#     authenticator = load_authenticator()
+
+# name, authentication_status, username = authenticator.login('Login', 'main')
+# auth_status = st.session_state.get('authentication_status')
+
+# if auth_status == True:
+#     is_state_initiaized = st.session_state.get('initialized',False)
+#     if not is_state_initiaized:
+#         print("Scenario page state reloaded")
+    
+initialize_data()
+
+st.subheader("Build response curves")
+    
+channels_list = st.session_state['channels_list']
+selected_channel_name = st.selectbox('Channel', st.session_state['channels_list'] + ['Others'], format_func=channel_name_formating,on_change=reset_curve_parameters)
+
+rcs = {}
+for channel_name in channels_list:
+    rcs[channel_name] = st.session_state['scenario'].channels[channel_name].response_curve_params
+# rcs = st.session_state['rcs']
+
+
+if 'K' not in st.session_state:
+    st.session_state['K'] = rcs[selected_channel_name]['K']
+if 'b' not in st.session_state:
+    st.session_state['b'] = rcs[selected_channel_name]['b']
+if 'a' not in st.session_state:
+    st.session_state['a'] = rcs[selected_channel_name]['a']
+if 'x0' not in st.session_state:
+    st.session_state['x0'] = rcs[selected_channel_name]['x0']
+    
+x = st.session_state['actual_input_df'][selected_channel_name].values
+y = st.session_state['actual_contribution_df'][selected_channel_name].values
+
+power = (np.ceil(np.log(x.max()) / np.log(10) )- 3)
+
+# fig = px.scatter(x, s_curve(x/10**power,
+#                             st.session_state['K'],
+#                             st.session_state['b'],
+#                             st.session_state['a'],
+#                             st.session_state['x0']))
+
+fig = px.scatter(x=x, y=y)
+fig.add_trace(go.Scatter(x=sorted(x), y=s_curve(sorted(x)/10**power,st.session_state['K'],
+                                    st.session_state['b'],
+                                    st.session_state['a'],
+                                    st.session_state['x0']),
+                        line=dict(color='red')))
+
+fig.update_layout(title_text="Response Curve",showlegend=False)
+fig.update_annotations(font_size=10)
+fig.update_xaxes(title='Spends')
+fig.update_yaxes(title='Revenue')
+
+st.plotly_chart(fig,use_container_width=True)
+
+r2 = r2_score(y, s_curve(x / 10**power, 
+                        st.session_state['K'],
+                        st.session_state['b'],
+                        st.session_state['a'],
+                        st.session_state['x0']))
+
+st.metric('R2',round(r2,2))
+columns = st.columns(4)
+
+with columns[0]:
+    st.number_input('K',key='K',format="%0.5f")
+with columns[1]:
+    st.number_input('b',key='b',format="%0.5f")
+with columns[2]:
+    st.number_input('a',key='a',step=0.0001,format="%0.5f")
+with columns[3]:
+    st.number_input('x0',key='x0',format="%0.5f")
+    
+
+st.button('Update parameters',on_click=update_response_curve)
+st.button('Reset parameters',on_click=reset_curve_parameters)
+scenario_name = st.text_input('Scenario name', key='scenario_input',placeholder='Scenario name',label_visibility='collapsed')
+st.button('Save', on_click=lambda  : save_scenario(scenario_name),disabled=len(st.session_state['scenario_input']) == 0)
+
+file_name = st.text_input('rcs download file name', key='file_name_input',placeholder='file name',label_visibility='collapsed')
+st.download_button(
+                    label="Download response curves",
+                    data=json.dumps(rcs),
+                    file_name=f"{file_name}.json",
+                    mime="application/json",
+                    disabled= len(file_name) == 0,
+                )
+
+
+def s_curve_derivative(x, K, b, a, x0):
+    # Derivative of the S-curve function
+    return a * b * K * np.exp(-a * (x - x0)) / ((1 + b * np.exp(-a * (x - x0))) ** 2)
+
+# Parameters of the S-curve
+K = st.session_state['K']
+b = st.session_state['b']
+a = st.session_state['a']
+x0 = st.session_state['x0']
+
+# Optimized spend value obtained from the tool
+optimized_spend = st.number_input('value of x')  # Replace this with your optimized spend value
+
+# Calculate the slope at the optimized spend value
+slope_at_optimized_spend = s_curve_derivative(optimized_spend, K, b, a, x0)
+
+st.write("Slope ", slope_at_optimized_spend)

pages/8_Scenario_Planner.py

@@ -0,0 +1,1133 @@
+import streamlit as st
+from numerize.numerize import numerize
+import numpy as np
+from functools import partial
+from collections import OrderedDict
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from utilities import (
+    format_numbers,
+    load_local_css,
+    set_header,
+    initialize_data,
+    load_authenticator,
+    send_email,
+    channel_name_formating,
+)
+from classes import class_from_dict, class_to_dict
+import pickle
+import streamlit_authenticator as stauth
+import yaml
+from yaml import SafeLoader
+import re
+import pandas as pd
+import plotly.express as px
+
+target = "Revenue"
+st.set_page_config(layout="wide")
+load_local_css("styles.css")
+set_header()
+
+for k, v in st.session_state.items():
+    if k not in ["logout", "login", "config"] and not k.startswith(
+        "FormSubmitter"
+    ):
+        st.session_state[k] = v
+# ======================================================== #
+# ======================= Functions ====================== #
+# ======================================================== #
+
+
+def optimize(key):
+    """
+    Optimize the spends for the sales
+    """
+
+    channel_list = [
+        key
+        for key, value in st.session_state["optimization_channels"].items()
+        if value
+    ]
+    # print('channel_list')
+    # print(channel_list)
+    # print('@@@@@@@@')
+    if len(channel_list) > 0:
+        scenario = st.session_state["scenario"]
+        if key.lower() == "spends":
+            with status_placeholder:
+                with st.spinner("Optimizing"):
+                    result = st.session_state["scenario"].optimize(
+                        st.session_state["total_spends_change"], channel_list
+                    )
+        elif key.lower() == "sales":
+            with status_placeholder:
+                with st.spinner("Optimizing"):
+
+                    result = st.session_state["scenario"].optimize_spends(
+                        st.session_state["total_sales_change"], channel_list
+                    )
+        for channel_name, modified_spends in result:
+
+            st.session_state[channel_name] = numerize(
+                modified_spends
+                * scenario.channels[channel_name].conversion_rate,
+                1,
+            )
+            prev_spends = (
+                st.session_state["scenario"]
+                .channels[channel_name]
+                .actual_total_spends
+            )
+            st.session_state[f"{channel_name}_change"] = round(
+                100 * (modified_spends - prev_spends) / prev_spends, 2
+            )
+
+
+def save_scenario(scenario_name):
+    """
+    Save the current scenario with the mentioned name in the session state
+
+    Parameters
+    ----------
+    scenario_name
+        Name of the scenario to be saved
+    """
+    if "saved_scenarios" not in st.session_state:
+        st.session_state = OrderedDict()
+
+    # st.session_state['saved_scenarios'][scenario_name] = st.session_state['scenario'].save()
+    st.session_state["saved_scenarios"][scenario_name] = class_to_dict(
+        st.session_state["scenario"]
+    )
+    st.session_state["scenario_input"] = ""
+    # print(type(st.session_state['saved_scenarios']))
+    with open("../saved_scenarios.pkl", "wb") as f:
+        pickle.dump(st.session_state["saved_scenarios"], f)
+
+
+def update_sales_abs():
+    actual_sales = _scenario.actual_total_sales
+    if validate_input(st.session_state["total_sales_change_abs"]):
+        modified_sales = extract_number_for_string(
+            st.session_state["total_sales_change_abs"]
+        )
+        st.session_state["total_sales_change"] = round(
+            ((modified_sales / actual_sales) - 1) * 100
+        )
+
+
+def update_sales():
+    st.session_state["total_sales_change_abs"] = numerize(
+        (1 + st.session_state["total_sales_change"] / 100)
+        * _scenario.actual_total_sales,
+        1,
+    )
+
+
+def update_all_spends_abs():
+    actual_spends = _scenario.actual_total_spends
+    if validate_input(st.session_state["total_spends_change_abs"]):
+        modified_spends = extract_number_for_string(
+            st.session_state["total_spends_change_abs"]
+        )
+        print(modified_spends)
+        print(actual_spends)
+
+        st.session_state["total_spends_change"] = (
+            (modified_spends / actual_spends) - 1
+        ) * 100
+
+        update_all_spends()
+
+
+def update_all_spends():
+    """
+    Updates spends for all the channels with the given overall spends change
+    """
+    percent_change = st.session_state["total_spends_change"]
+    st.session_state["total_spends_change_abs"] = numerize(
+        (1 + percent_change / 100) * _scenario.actual_total_spends, 1
+    )
+    for channel_name in st.session_state["channels_list"]:
+        channel = st.session_state["scenario"].channels[channel_name]
+        current_spends = channel.actual_total_spends
+        modified_spends = (1 + percent_change / 100) * current_spends
+        st.session_state["scenario"].update(channel_name, modified_spends)
+        st.session_state[channel_name] = numerize(
+            modified_spends * channel.conversion_rate, 1
+        )
+        st.session_state[f"{channel_name}_change"] = percent_change
+
+
+def extract_number_for_string(string_input):
+    string_input = string_input.upper()
+    if string_input.endswith("K"):
+        return float(string_input[:-1]) * 10**3
+    elif string_input.endswith("M"):
+        return float(string_input[:-1]) * 10**6
+    elif string_input.endswith("B"):
+        return float(string_input[:-1]) * 10**9
+
+
+def validate_input(string_input):
+    pattern = r"\d+\.?\d*[K|M|B]$"
+    match = re.match(pattern, string_input)
+    if match is None:
+        return False
+    return True
+
+
+def update_data_by_percent(channel_name):
+    prev_spends = (
+        st.session_state["scenario"].channels[channel_name].actual_total_spends
+        * st.session_state["scenario"].channels[channel_name].conversion_rate
+    )
+    modified_spends = prev_spends * (
+        1 + st.session_state[f"{channel_name}_change"] / 100
+    )
+    st.session_state[channel_name] = numerize(modified_spends, 1)
+    st.session_state["scenario"].update(
+        channel_name,
+        modified_spends
+        / st.session_state["scenario"].channels[channel_name].conversion_rate,
+    )
+
+
+def update_data(channel_name):
+    """
+    Updates the spends for the given channel
+    """
+
+    if validate_input(st.session_state[channel_name]):
+        modified_spends = extract_number_for_string(
+            st.session_state[channel_name]
+        )
+        prev_spends = (
+            st.session_state["scenario"]
+            .channels[channel_name]
+            .actual_total_spends
+            * st.session_state["scenario"]
+            .channels[channel_name]
+            .conversion_rate
+        )
+        st.session_state[f"{channel_name}_change"] = round(
+            100 * (modified_spends - prev_spends) / prev_spends, 2
+        )
+        st.session_state["scenario"].update(
+            channel_name,
+            modified_spends
+            / st.session_state["scenario"]
+            .channels[channel_name]
+            .conversion_rate,
+        )
+    # st.session_state['scenario'].update(channel_name, modified_spends)
+    # else:
+    #     try:
+    #         modified_spends = float(st.session_state[channel_name])
+    #         prev_spends = st.session_state['scenario'].channels[channel_name].actual_total_spends * st.session_state['scenario'].channels[channel_name].conversion_rate
+    #         st.session_state[f'{channel_name}_change'] = round(100*(modified_spends - prev_spends) / prev_spends,2)
+    #         st.session_state['scenario'].update(channel_name, modified_spends/st.session_state['scenario'].channels[channel_name].conversion_rate)
+    #         st.session_state[f'{channel_name}'] = numerize(modified_spends,1)
+    #     except ValueError:
+    #         st.write('Invalid input')
+
+
+def select_channel_for_optimization(channel_name):
+    """
+    Marks the given channel for optimization
+    """
+    st.session_state["optimization_channels"][channel_name] = st.session_state[
+        f"{channel_name}_selected"
+    ]
+
+
+def select_all_channels_for_optimization():
+    """
+    Marks all the channel for optimization
+    """
+    for channel_name in st.session_state["optimization_channels"].keys():
+        st.session_state[f"{channel_name}_selected"] = st.session_state[
+            "optimze_all_channels"
+        ]
+        st.session_state["optimization_channels"][channel_name] = (
+            st.session_state["optimze_all_channels"]
+        )
+
+
+def update_penalty():
+    """
+    Updates the penalty flag for sales calculation
+    """
+    st.session_state["scenario"].update_penalty(
+        st.session_state["apply_penalty"]
+    )
+
+
+def reset_scenario():
+    # #print(st.session_state['default_scenario_dict'])
+    # st.session_state['scenario']  = class_from_dict(st.session_state['default_scenario_dict'])
+    # for channel in st.session_state['scenario'].channels.values():
+    #     st.session_state[channel.name] = float(channel.actual_total_spends * channel.conversion_rate)
+    initialize_data()
+    for channel_name in st.session_state["channels_list"]:
+        st.session_state[f"{channel_name}_selected"] = False
+        st.session_state[f"{channel_name}_change"] = 0
+    st.session_state["optimze_all_channels"] = False
+
+
+def format_number(num):
+    if num >= 1_000_000:
+        return f"{num / 1_000_000:.2f}M"
+    elif num >= 1_000:
+        return f"{num / 1_000:.0f}K"
+    else:
+        return f"{num:.2f}"
+
+
+def summary_plot(data, x, y, title, text_column):
+    fig = px.bar(
+        data,
+        x=x,
+        y=y,
+        orientation="h",
+        title=title,
+        text=text_column,
+        color="Channel_name",
+    )
+
+    # Convert text_column to numeric values
+    data[text_column] = pd.to_numeric(data[text_column], errors="coerce")
+
+    # Update the format of the displayed text based on magnitude
+    fig.update_traces(
+        texttemplate="%{text:.2s}",
+        textposition="outside",
+        hovertemplate="%{x:.2s}",
+    )
+
+    fig.update_layout(
+        xaxis_title=x, yaxis_title="Channel Name", showlegend=False
+    )
+    return fig
+
+
+def s_curve(x, K, b, a, x0):
+    return K / (1 + b * np.exp(-a * (x - x0)))
+
+
+def find_segment_value(x, roi, mroi):
+    start_value = x[0]
+    end_value = x[len(x) - 1]
+
+    # Condition for green region: Both MROI and ROI > 1
+    green_condition = (roi > 1) & (mroi > 1)
+    left_indices = np.where(green_condition)[0]
+    left_value = x[left_indices[0]] if left_indices.size > 0 else x[0]
+
+    right_indices = np.where(green_condition)[0]
+    right_value = x[right_indices[-1]] if right_indices.size > 0 else x[0]
+
+    return start_value, end_value, left_value, right_value
+
+
+def calculate_rgba(
+    start_value, end_value, left_value, right_value, current_channel_spends
+):
+    # Initialize alpha to None for clarity
+    alpha = None
+
+    # Determine the color and calculate relative_position and alpha based on the point's position
+    if start_value <= current_channel_spends <= left_value:
+        color = "yellow"
+        relative_position = (current_channel_spends - start_value) / (
+            left_value - start_value
+        )
+        alpha = 0.8 - (
+            0.6 * relative_position
+        )  # Alpha decreases from start to end
+
+    elif left_value < current_channel_spends <= right_value:
+        color = "green"
+        relative_position = (current_channel_spends - left_value) / (
+            right_value - left_value
+        )
+        alpha = 0.8 - (
+            0.6 * relative_position
+        )  # Alpha decreases from start to end
+
+    elif right_value < current_channel_spends <= end_value:
+        color = "red"
+        relative_position = (current_channel_spends - right_value) / (
+            end_value - right_value
+        )
+        alpha = 0.2 + (
+            0.6 * relative_position
+        )  # Alpha increases from start to end
+
+    else:
+        # Default case, if the spends are outside the defined ranges
+        return "rgba(136, 136, 136, 0.5)"  # Grey for values outside the range
+
+    # Ensure alpha is within the intended range in case of any calculation overshoot
+    alpha = max(0.2, min(alpha, 0.8))
+
+    # Define color codes for RGBA
+    color_codes = {
+        "yellow": "255, 255, 0",  # RGB for yellow
+        "green": "0, 128, 0",  # RGB for green
+        "red": "255, 0, 0",  # RGB for red
+    }
+
+    rgba = f"rgba({color_codes[color]}, {alpha})"
+    return rgba
+
+
+def debug_temp(x_test, power, K, b, a, x0):
+    print("*" * 100)
+    # Calculate the count of bins
+    count_lower_bin = sum(1 for x in x_test if x <= 2524)
+    count_center_bin = sum(1 for x in x_test if x > 2524 and x <= 3377)
+    count_ = sum(1 for x in x_test if x > 3377)
+
+    print(
+        f"""
+            lower : {count_lower_bin}
+            center : {count_center_bin}
+            upper : {count_}
+          """
+    )
+
+
+# @st.cache
+def plot_response_curves():
+    cols = 4
+    rows = (
+        len(channels_list) // cols
+        if len(channels_list) % cols == 0
+        else len(channels_list) // cols + 1
+    )
+    rcs = st.session_state["rcs"]
+    shapes = []
+    fig = make_subplots(rows=rows, cols=cols, subplot_titles=channels_list)
+    for i in range(0, len(channels_list)):
+        col = channels_list[i]
+        x_actual = st.session_state["scenario"].channels[col].actual_spends
+        # x_modified = st.session_state["scenario"].channels[col].modified_spends
+
+        power = np.ceil(np.log(x_actual.max()) / np.log(10)) - 3
+
+        K = rcs[col]["K"]
+        b = rcs[col]["b"]
+        a = rcs[col]["a"]
+        x0 = rcs[col]["x0"]
+
+        x_plot = np.linspace(0, 5 * x_actual.sum(), 50)
+
+        x, y, marginal_roi = [], [], []
+        for x_p in x_plot:
+            x.append(x_p * x_actual / x_actual.sum())
+
+        for index in range(len(x_plot)):
+            y.append(s_curve(x[index] / 10**power, K, b, a, x0))
+
+        for index in range(len(x_plot)):
+            marginal_roi.append(
+                a
+                * y[index]
+                * (1 - y[index] / np.maximum(K, np.finfo(float).eps))
+            )
+
+        x = (
+            np.sum(x, axis=1)
+            * st.session_state["scenario"].channels[col].conversion_rate
+        )
+        y = np.sum(y, axis=1)
+        marginal_roi = (
+            np.average(marginal_roi, axis=1)
+            / st.session_state["scenario"].channels[col].conversion_rate
+        )
+
+        roi = y / np.maximum(x, np.finfo(float).eps)
+
+        fig.add_trace(
+            go.Scatter(
+                x=x,
+                y=y,
+                name=col,
+                customdata=np.stack((roi, marginal_roi), axis=-1),
+                hovertemplate="Spend:%{x:$.2s}<br>Sale:%{y:$.2s}<br>ROI:%{customdata[0]:.3f}<br>MROI:%{customdata[1]:.3f}",
+                line=dict(color="blue"),
+            ),
+            row=1 + (i) // cols,
+            col=i % cols + 1,
+        )
+
+        x_optimal = (
+            st.session_state["scenario"].channels[col].modified_total_spends
+            * st.session_state["scenario"].channels[col].conversion_rate
+        )
+        y_optimal = (
+            st.session_state["scenario"].channels[col].modified_total_sales
+        )
+
+        # if col == "Paid_social_others":
+        #     debug_temp(x_optimal * x_actual / x_actual.sum(), power, K, b, a, x0)
+
+        fig.add_trace(
+            go.Scatter(
+                x=[x_optimal],
+                y=[y_optimal],
+                name=col,
+                legendgroup=col,
+                showlegend=False,
+                marker=dict(color=["black"]),
+            ),
+            row=1 + (i) // cols,
+            col=i % cols + 1,
+        )
+
+        shapes.append(
+            go.layout.Shape(
+                type="line",
+                x0=0,
+                y0=y_optimal,
+                x1=x_optimal,
+                y1=y_optimal,
+                line_width=1,
+                line_dash="dash",
+                line_color="black",
+                xref=f"x{i+1}",
+                yref=f"y{i+1}",
+            )
+        )
+
+        shapes.append(
+            go.layout.Shape(
+                type="line",
+                x0=x_optimal,
+                y0=0,
+                x1=x_optimal,
+                y1=y_optimal,
+                line_width=1,
+                line_dash="dash",
+                line_color="black",
+                xref=f"x{i+1}",
+                yref=f"y{i+1}",
+            )
+        )
+
+        start_value, end_value, left_value, right_value = find_segment_value(
+            x,
+            roi,
+            marginal_roi,
+        )
+
+        # Adding background colors
+        y_max = y.max() * 1.3  # 30% extra space above the max
+
+        # Yellow region
+        shapes.append(
+            go.layout.Shape(
+                type="rect",
+                x0=start_value,
+                y0=0,
+                x1=left_value,
+                y1=y_max,
+                line=dict(width=0),
+                fillcolor="rgba(255, 255, 0, 0.3)",
+                layer="below",
+                xref=f"x{i+1}",
+                yref=f"y{i+1}",
+            )
+        )
+
+        # Green region
+        shapes.append(
+            go.layout.Shape(
+                type="rect",
+                x0=left_value,
+                y0=0,
+                x1=right_value,
+                y1=y_max,
+                line=dict(width=0),
+                fillcolor="rgba(0, 255, 0, 0.3)",
+                layer="below",
+                xref=f"x{i+1}",
+                yref=f"y{i+1}",
+            )
+        )
+
+        # Red region
+        shapes.append(
+            go.layout.Shape(
+                type="rect",
+                x0=right_value,
+                y0=0,
+                x1=end_value,
+                y1=y_max,
+                line=dict(width=0),
+                fillcolor="rgba(255, 0, 0, 0.3)",
+                layer="below",
+                xref=f"x{i+1}",
+                yref=f"y{i+1}",
+            )
+        )
+
+    fig.update_layout(
+        # height=1000,
+        # width=1000,
+        title_text="Response Curves (X: Spends Vs Y: Revenue)",
+        showlegend=False,
+        shapes=shapes,
+    )
+    fig.update_annotations(font_size=10)
+    # fig.update_xaxes(title="Spends")
+    # fig.update_yaxes(title=target)
+    fig.update_yaxes(
+        gridcolor="rgba(136, 136, 136, 0.5)", gridwidth=0.5, griddash="dash"
+    )
+
+    return fig
+
+
+# @st.cache
+# def plot_response_curves():
+#     cols = 4
+#     rcs = st.session_state["rcs"]
+#     shapes = []
+#     fig = make_subplots(rows=6, cols=cols, subplot_titles=channels_list)
+#     for i in range(0, len(channels_list)):
+#         col = channels_list[i]
+#         x = st.session_state["actual_df"][col].values
+#         spends = x.sum()
+#         power = np.ceil(np.log(x.max()) / np.log(10)) - 3
+#         x = np.linspace(0, 3 * x.max(), 200)
+
+#         K = rcs[col]["K"]
+#         b = rcs[col]["b"]
+#         a = rcs[col]["a"]
+#         x0 = rcs[col]["x0"]
+
+#         y = s_curve(x / 10**power, K, b, a, x0)
+#         roi = y / x
+#         marginal_roi = a * (y) * (1 - y / K)
+#         fig.add_trace(
+#             go.Scatter(
+#                 x=52
+#                 * x
+#                 * st.session_state["scenario"].channels[col].conversion_rate,
+#                 y=52 * y,
+#                 name=col,
+#                 customdata=np.stack((roi, marginal_roi), axis=-1),
+#                 hovertemplate="Spend:%{x:$.2s}<br>Sale:%{y:$.2s}<br>ROI:%{customdata[0]:.3f}<br>MROI:%{customdata[1]:.3f}",
+#             ),
+#             row=1 + (i) // cols,
+#             col=i % cols + 1,
+#         )
+
+#         fig.add_trace(
+#             go.Scatter(
+#                 x=[
+#                     spends
+#                     * st.session_state["scenario"]
+#                     .channels[col]
+#                     .conversion_rate
+#                 ],
+#                 y=[52 * s_curve(spends / (10**power * 52), K, b, a, x0)],
+#                 name=col,
+#                 legendgroup=col,
+#                 showlegend=False,
+#                 marker=dict(color=["black"]),
+#             ),
+#             row=1 + (i) // cols,
+#             col=i % cols + 1,
+#         )
+
+#         shapes.append(
+#             go.layout.Shape(
+#                 type="line",
+#                 x0=0,
+#                 y0=52 * s_curve(spends / (10**power * 52), K, b, a, x0),
+#                 x1=spends
+#                 * st.session_state["scenario"].channels[col].conversion_rate,
+#                 y1=52 * s_curve(spends / (10**power * 52), K, b, a, x0),
+#                 line_width=1,
+#                 line_dash="dash",
+#                 line_color="black",
+#                 xref=f"x{i+1}",
+#                 yref=f"y{i+1}",
+#             )
+#         )
+
+#         shapes.append(
+#             go.layout.Shape(
+#                 type="line",
+#                 x0=spends
+#                 * st.session_state["scenario"].channels[col].conversion_rate,
+#                 y0=0,
+#                 x1=spends
+#                 * st.session_state["scenario"].channels[col].conversion_rate,
+#                 y1=52 * s_curve(spends / (10**power * 52), K, b, a, x0),
+#                 line_width=1,
+#                 line_dash="dash",
+#                 line_color="black",
+#                 xref=f"x{i+1}",
+#                 yref=f"y{i+1}",
+#             )
+#         )
+
+#     fig.update_layout(
+#         height=1500,
+#         width=1000,
+#         title_text="Response Curves",
+#         showlegend=False,
+#         shapes=shapes,
+#     )
+#     fig.update_annotations(font_size=10)
+#     fig.update_xaxes(title="Spends")
+#     fig.update_yaxes(title=target)
+#     return fig
+
+
+# ======================================================== #
+# ==================== HTML Components =================== #
+# ======================================================== #
+
+
+def generate_spending_header(heading):
+    return st.markdown(
+        f"""<h2 class="spends-header">{heading}</h2>""", unsafe_allow_html=True
+    )
+
+
+# ======================================================== #
+# =================== Session variables ================== #
+# ======================================================== #
+
+with open("config.yaml") as file:
+    config = yaml.load(file, Loader=SafeLoader)
+    st.session_state["config"] = config
+
+authenticator = stauth.Authenticate(
+    config["credentials"],
+    config["cookie"]["name"],
+    config["cookie"]["key"],
+    config["cookie"]["expiry_days"],
+    config["preauthorized"],
+)
+st.session_state["authenticator"] = authenticator
+name, authentication_status, username = authenticator.login("Login", "main")
+auth_status = st.session_state.get("authentication_status")
+if auth_status == True:
+    authenticator.logout("Logout", "main")
+    is_state_initiaized = st.session_state.get("initialized", False)
+    if not is_state_initiaized:
+        initialize_data()
+
+    channels_list = st.session_state["channels_list"]
+
+    # ======================================================== #
+    # ========================== UI ========================== #
+    # ======================================================== #
+
+    # print(list(st.session_state.keys()))
+
+    st.header("Simulation")
+    main_header = st.columns((2, 2))
+    sub_header = st.columns((1, 1, 1, 1))
+    _scenario = st.session_state["scenario"]
+
+    if "total_spends_change_abs" not in st.session_state:
+        st.session_state["total_spends_change_abs"] = numerize(
+            _scenario.actual_total_spends, 1
+        )
+
+    if "total_sales_change_abs" not in st.session_state:
+        st.session_state["total_sales_change_abs"] = numerize(
+            _scenario.actual_total_sales, 1
+        )
+
+    with main_header[0]:
+        st.subheader("Actual")
+
+    with main_header[-1]:
+        st.subheader("Simulated")
+
+    with sub_header[0]:
+        st.metric(
+            label="Spends", value=format_numbers(_scenario.actual_total_spends)
+        )
+
+    with sub_header[1]:
+        st.metric(
+            label=target,
+            value=format_numbers(
+                float(_scenario.actual_total_sales), include_indicator=False
+            ),
+        )
+
+    with sub_header[2]:
+        st.metric(
+            label="Spends",
+            value=format_numbers(_scenario.modified_total_spends),
+            delta=numerize(_scenario.delta_spends, 1),
+        )
+
+    with sub_header[3]:
+        st.metric(
+            label=target,
+            value=format_numbers(
+                float(_scenario.modified_total_sales), include_indicator=False
+            ),
+            delta=numerize(_scenario.delta_sales, 1),
+        )
+
+    with st.expander("Channel Spends Simulator"):
+        _columns1 = st.columns((2, 2, 1, 1))
+        with _columns1[0]:
+
+            optimization_selection = st.selectbox(
+                "Optimize", options=["Spends", "Sales"], key="optimization_key"
+            )
+        with _columns1[1]:
+            st.markdown("#")
+            st.checkbox(
+                label="Optimize all Channels",
+                key=f"optimze_all_channels",
+                value=False,
+                on_change=select_all_channels_for_optimization,
+            )
+
+        with _columns1[2]:
+            st.markdown("#")
+            st.button(
+                "Optimize",
+                on_click=optimize,
+                args=(st.session_state["optimization_key"],),
+            )
+
+        with _columns1[3]:
+            st.markdown("#")
+            st.button("Reset", on_click=reset_scenario)
+
+        _columns2 = st.columns((2, 2, 2))
+        if st.session_state["optimization_key"] == "Spends":
+            with _columns2[0]:
+                spend_input = st.text_input(
+                    "Absolute",
+                    key="total_spends_change_abs",
+                    # label_visibility="collapsed",
+                    on_change=update_all_spends_abs,
+                )
+            with _columns2[1]:
+
+                st.number_input(
+                    "Percent",
+                    key=f"total_spends_change",
+                    step=1,
+                    on_change=update_all_spends,
+                )
+        elif st.session_state["optimization_key"] == "Sales":
+            with _columns2[0]:
+
+                sales_input = st.text_input(
+                    "Absolute",
+                    key="total_sales_change_abs",
+                    on_change=update_sales_abs,
+                )
+            with _columns2[1]:
+                st.number_input(
+                    "Percent change",
+                    key=f"total_sales_change",
+                    step=1,
+                    on_change=update_sales,
+                )
+
+        with _columns2[2]:
+            st.markdown("#")
+            status_placeholder = st.empty()
+
+        st.markdown(
+            """<hr class="spends-heading-seperator">""", unsafe_allow_html=True
+        )
+        _columns = st.columns((2.5, 2, 1.5, 1.5, 1))
+        with _columns[0]:
+            generate_spending_header("Channel")
+        with _columns[1]:
+            generate_spending_header("Spends Input")
+        with _columns[2]:
+            generate_spending_header("Spends")
+        with _columns[3]:
+            generate_spending_header(target)
+        with _columns[4]:
+            generate_spending_header("Optimize")
+
+        st.markdown(
+            """<hr class="spends-heading-seperator">""", unsafe_allow_html=True
+        )
+
+        if "acutual_predicted" not in st.session_state:
+            st.session_state["acutual_predicted"] = {
+                "Channel_name": [],
+                "Actual_spend": [],
+                "Optimized_spend": [],
+                "Delta": [],
+            }
+        for i, channel_name in enumerate(channels_list):
+            _channel_class = st.session_state["scenario"].channels[
+                channel_name
+            ]
+            _columns = st.columns((2.5, 1.5, 1.5, 1.5, 1))
+            with _columns[0]:
+                st.write(channel_name_formating(channel_name))
+                bin_placeholder = st.container()
+
+            with _columns[1]:
+                channel_bounds = _channel_class.bounds
+                channel_spends = float(_channel_class.actual_total_spends)
+                min_value = float(
+                    (1 + channel_bounds[0] / 100) * channel_spends
+                )
+                max_value = float(
+                    (1 + channel_bounds[1] / 100) * channel_spends
+                )
+                ##print(st.session_state[channel_name])
+                spend_input = st.text_input(
+                    channel_name,
+                    key=channel_name,
+                    label_visibility="collapsed",
+                    on_change=partial(update_data, channel_name),
+                )
+                if not validate_input(spend_input):
+                    st.error("Invalid input")
+
+                st.number_input(
+                    "Percent change",
+                    key=f"{channel_name}_change",
+                    step=1,
+                    on_change=partial(update_data_by_percent, channel_name),
+                )
+
+            with _columns[2]:
+                # spends
+                current_channel_spends = float(
+                    _channel_class.modified_total_spends
+                    * _channel_class.conversion_rate
+                )
+                actual_channel_spends = float(
+                    _channel_class.actual_total_spends
+                    * _channel_class.conversion_rate
+                )
+                spends_delta = float(
+                    _channel_class.delta_spends
+                    * _channel_class.conversion_rate
+                )
+                st.session_state["acutual_predicted"]["Channel_name"].append(
+                    channel_name
+                )
+                st.session_state["acutual_predicted"]["Actual_spend"].append(
+                    actual_channel_spends
+                )
+                st.session_state["acutual_predicted"][
+                    "Optimized_spend"
+                ].append(current_channel_spends)
+                st.session_state["acutual_predicted"]["Delta"].append(
+                    spends_delta
+                )
+                ## REMOVE
+                st.metric(
+                    "Spends",
+                    format_numbers(current_channel_spends),
+                    delta=numerize(spends_delta, 1),
+                    label_visibility="collapsed",
+                )
+
+            with _columns[3]:
+                # sales
+                current_channel_sales = float(
+                    _channel_class.modified_total_sales
+                )
+                actual_channel_sales = float(_channel_class.actual_total_sales)
+                sales_delta = float(_channel_class.delta_sales)
+                st.metric(
+                    target,
+                    format_numbers(
+                        current_channel_sales, include_indicator=False
+                    ),
+                    delta=numerize(sales_delta, 1),
+                    label_visibility="collapsed",
+                )
+
+            with _columns[4]:
+
+                st.checkbox(
+                    label="select for optimization",
+                    key=f"{channel_name}_selected",
+                    value=False,
+                    on_change=partial(
+                        select_channel_for_optimization, channel_name
+                    ),
+                    label_visibility="collapsed",
+                )
+
+            st.markdown(
+                """<hr class="spends-child-seperator">""",
+                unsafe_allow_html=True,
+            )
+
+            # Bins
+            col = channels_list[i]
+            x_actual = st.session_state["scenario"].channels[col].actual_spends
+            x_modified = (
+                st.session_state["scenario"].channels[col].modified_spends
+            )
+
+            x_total = x_modified.sum()
+            power = np.ceil(np.log(x_actual.max()) / np.log(10)) - 3
+
+            K = st.session_state["rcs"][col]["K"]
+            b = st.session_state["rcs"][col]["b"]
+            a = st.session_state["rcs"][col]["a"]
+            x0 = st.session_state["rcs"][col]["x0"]
+
+            x_plot = np.linspace(0, 5 * x_actual.sum(), 200)
+
+            x, y, marginal_roi = [], [], []
+            for x_p in x_plot:
+                x.append(x_p * x_actual / x_actual.sum())
+
+            for index in range(len(x_plot)):
+                y.append(s_curve(x[index] / 10**power, K, b, a, x0))
+
+            for index in range(len(x_plot)):
+                marginal_roi.append(
+                    a
+                    * y[index]
+                    * (1 - y[index] / np.maximum(K, np.finfo(float).eps))
+                )
+
+            x = (
+                np.sum(x, axis=1)
+                * st.session_state["scenario"].channels[col].conversion_rate
+            )
+            y = np.sum(y, axis=1)
+            marginal_roi = (
+                np.average(marginal_roi, axis=1)
+                / st.session_state["scenario"].channels[col].conversion_rate
+            )
+
+            roi = y / np.maximum(x, np.finfo(float).eps)
+
+            start_value, end_value, left_value, right_value = (
+                find_segment_value(
+                    x,
+                    roi,
+                    marginal_roi,
+                )
+            )
+
+            rgba = calculate_rgba(
+                start_value,
+                end_value,
+                left_value,
+                right_value,
+                current_channel_spends,
+            )
+
+            # Protecting division by zero by adding a small epsilon to denominators
+            roi_current = current_channel_sales / np.maximum(
+                current_channel_spends, np.finfo(float).eps
+            )
+            marginal_roi_current = (
+                st.session_state["scenario"]
+                .channels[col]
+                .get_marginal_roi("modified")
+            )
+
+            with bin_placeholder:
+                st.markdown(
+                    f"""
+                    <div style="
+                        border-radius: 12px;
+                        background-color: {rgba};
+                        padding: 10px;
+                        text-align: center;
+                        color: #006EC0;
+                        ">
+                        <p style="margin: 0; font-size: 20px;">ROI: {round(roi_current,1)}</p>
+                        <p style="margin: 0; font-size: 20px;">Marginal ROI: {round(marginal_roi_current,1)}</p>
+                    </div>
+                    """,
+                    unsafe_allow_html=True,
+                )
+
+    with st.expander("See Response Curves"):
+        fig = plot_response_curves()
+        st.plotly_chart(fig, use_container_width=True)
+
+    _columns = st.columns(2)
+    with _columns[0]:
+        st.subheader("Save Scenario")
+        scenario_name = st.text_input(
+            "Scenario name",
+            key="scenario_input",
+            placeholder="Scenario name",
+            label_visibility="collapsed",
+        )
+        st.button(
+            "Save",
+            on_click=lambda: save_scenario(scenario_name),
+            disabled=len(st.session_state["scenario_input"]) == 0,
+        )
+
+    summary_df = pd.DataFrame(st.session_state["acutual_predicted"])
+    summary_df.drop_duplicates(
+        subset="Channel_name", keep="last", inplace=True
+    )
+
+    summary_df_sorted = summary_df.sort_values(by="Delta", ascending=False)
+    summary_df_sorted["Delta_percent"] = np.round(
+        (
+            (
+                summary_df_sorted["Optimized_spend"]
+                / summary_df_sorted["Actual_spend"]
+            )
+            - 1
+        )
+        * 100,
+        2,
+    )
+
+    with open("summary_df.pkl", "wb") as f:
+        pickle.dump(summary_df_sorted, f)
+        # st.dataframe(summary_df_sorted)
+        # ___columns=st.columns(3)
+        # with ___columns[2]:
+        #     fig=summary_plot(summary_df_sorted, x='Delta_percent', y='Channel_name', title='Delta', text_column='Delta_percent')
+        #     st.plotly_chart(fig,use_container_width=True)
+        # with ___columns[0]:
+        #     fig=summary_plot(summary_df_sorted, x='Actual_spend', y='Channel_name', title='Actual Spend', text_column='Actual_spend')
+        #     st.plotly_chart(fig,use_container_width=True)
+        # with ___columns[1]:
+        #     fig=summary_plot(summary_df_sorted, x='Optimized_spend', y='Channel_name', title='Planned Spend', text_column='Optimized_spend')
+        #     st.plotly_chart(fig,use_container_width=True)
+
+elif auth_status == False:
+    st.error("Username/Password is incorrect")
+
+if auth_status != True:
+    try:
+        username_forgot_pw, email_forgot_password, random_password = (
+            authenticator.forgot_password("Forgot password")
+        )
+        if username_forgot_pw:
+            st.session_state["config"]["credentials"]["usernames"][
+                username_forgot_pw
+            ]["password"] = stauth.Hasher([random_password]).generate()[0]
+            send_email(email_forgot_password, random_password)
+            st.success("New password sent securely")
+            # Random password to be transferred to user securely
+        elif username_forgot_pw == False:
+            st.error("Username not found")
+    except Exception as e:
+        st.error(e)

pages/9_Saved_Scenarios.py

@@ -0,0 +1,276 @@
+import streamlit as st
+from numerize.numerize import numerize
+import io
+import pandas as pd
+from utilities import (format_numbers,decimal_formater,
+                       channel_name_formating,
+                       load_local_css,set_header,
+                       initialize_data,
+                       load_authenticator)
+from openpyxl import Workbook
+from openpyxl.styles import Alignment,Font,PatternFill
+import pickle
+import streamlit_authenticator as stauth
+import yaml
+from yaml import SafeLoader
+from classes import class_from_dict
+
+st.set_page_config(layout='wide')
+load_local_css('styles.css')
+set_header()
+
+# for k, v in st.session_state.items():
+#     if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+#         st.session_state[k] = v
+
+def create_scenario_summary(scenario_dict):
+    summary_rows = []
+    for channel_dict in scenario_dict['channels']:
+        name_mod = channel_name_formating(channel_dict['name'])
+        summary_rows.append([name_mod,
+                             channel_dict.get('actual_total_spends') * channel_dict.get('conversion_rate'),
+                             channel_dict.get('modified_total_spends') * channel_dict.get('conversion_rate'),
+                             channel_dict.get('actual_total_sales') ,
+                             channel_dict.get('modified_total_sales'),
+                             channel_dict.get('actual_total_sales') / (channel_dict.get('actual_total_spends') * channel_dict.get('conversion_rate')), 
+                             channel_dict.get('modified_total_sales') / (channel_dict.get('modified_total_spends') * channel_dict.get('conversion_rate')),
+                             channel_dict.get('actual_mroi'), 
+                             channel_dict.get('modified_mroi'),
+                             channel_dict.get('actual_total_spends') * channel_dict.get('conversion_rate') / channel_dict.get('actual_total_sales'),
+                             channel_dict.get('modified_total_spends') * channel_dict.get('conversion_rate') / channel_dict.get('modified_total_sales')])
+        
+    summary_rows.append(['Total',
+                         scenario_dict.get('actual_total_spends'), 
+                         scenario_dict.get('modified_total_spends'),
+                         scenario_dict.get('actual_total_sales'), 
+                         scenario_dict.get('modified_total_sales'),
+                         scenario_dict.get('actual_total_sales') / scenario_dict.get('actual_total_spends'),
+                         scenario_dict.get('modified_total_sales') / scenario_dict.get('modified_total_spends'),
+                         '-',
+                         '-',
+                         scenario_dict.get('actual_total_spends') / scenario_dict.get('actual_total_sales'),
+                         scenario_dict.get('modified_total_spends') / scenario_dict.get('modified_total_sales')])
+    
+    columns_index = pd.MultiIndex.from_product([[''],['Channel']], names=["first", "second"])
+    columns_index = columns_index.append(pd.MultiIndex.from_product([['Spends','NRPU','ROI','MROI','Spend per NRPU'],['Actual','Simulated']], names=["first", "second"]))
+    return  pd.DataFrame(summary_rows, columns=columns_index)
+    
+
+   
+def summary_df_to_worksheet(df, ws):
+    heading_fill = PatternFill(fill_type='solid',start_color='FF11B6BD',end_color='FF11B6BD')
+    for j,header in enumerate(df.columns.values):
+        col = j + 1
+        for i in range(1,3):
+            ws.cell(row=i, column=j + 1, value=header[i - 1]).font = Font(bold=True, color='FF11B6BD')
+            ws.cell(row=i,column=j+1).fill = heading_fill
+        if  col > 1 and (col - 6)%5==0:    
+            ws.merge_cells(start_row=1, end_row=1, start_column = col-3, end_column=col)
+            ws.cell(row=1,column=col).alignment = Alignment(horizontal='center')
+    for i,row in enumerate(df.itertuples()):
+        for j,value in enumerate(row):
+            if j == 0:
+                continue
+            elif (j-2)%4 == 0 or (j-3)%4 == 0:
+                ws.cell(row=i+3, column = j, value=value).number_format = '$#,##0.0' 
+            else:
+                ws.cell(row=i+3, column = j, value=value)
+   
+from openpyxl.utils import get_column_letter
+from openpyxl.styles import Font, PatternFill
+import logging
+
+def scenario_df_to_worksheet(df, ws):
+    heading_fill = PatternFill(start_color='FF11B6BD', end_color='FF11B6BD', fill_type='solid')
+    
+    for j, header in enumerate(df.columns.values):
+        cell = ws.cell(row=1, column=j + 1, value=header)
+        cell.font = Font(bold=True, color='FF11B6BD')
+        cell.fill = heading_fill
+
+    for i, row in enumerate(df.itertuples()):
+        for j, value in enumerate(row[1:], start=1):  # Start from index 1 to skip the index column
+            try:
+                cell = ws.cell(row=i + 2, column=j, value=value)
+                if isinstance(value, (int, float)):
+                    cell.number_format = '$#,##0.0'
+                elif isinstance(value, str):
+                    cell.value = value[:32767]
+                else:
+                    cell.value = str(value)
+            except ValueError as e:
+                logging.error(f"Error assigning value '{value}' to cell {get_column_letter(j)}{i+2}: {e}")
+                cell.value = None  # Assign None to the cell where the error occurred
+
+    return ws
+
+
+
+
+
+   
+def download_scenarios():
+    """
+    Makes a excel with all saved scenarios and saves it locally
+    """
+    ## create summary page
+    if len(scenarios_to_download) == 0:
+        return
+    wb = Workbook()
+    wb.iso_dates = True
+    wb.remove(wb.active)
+    st.session_state['xlsx_buffer'] = io.BytesIO()
+    summary_df = None
+    #print(scenarios_to_download)
+    for scenario_name in scenarios_to_download:
+        scenario_dict =  st.session_state['saved_scenarios'][scenario_name]
+        _spends = []
+        column_names = ['Date']
+        _sales = None
+        dates = None
+        summary_rows = []
+        for channel in scenario_dict['channels']:
+            if dates is None:
+                dates = channel.get('dates')
+                _spends.append(dates)
+            if _sales is None:
+                _sales = channel.get('modified_sales')
+            else:
+                _sales += channel.get('modified_sales')
+            _spends.append(channel.get('modified_spends') * channel.get('conversion_rate'))
+            column_names.append(channel.get('name'))
+            
+            name_mod = channel_name_formating(channel['name'])
+            summary_rows.append([name_mod,
+                                channel.get('modified_total_spends') * channel.get('conversion_rate') ,
+                                channel.get('modified_total_sales'),
+                                channel.get('modified_total_sales') / channel.get('modified_total_spends') * channel.get('conversion_rate'),
+                                channel.get('modified_mroi'),
+                                channel.get('modified_total_sales') / channel.get('modified_total_spends') * channel.get('conversion_rate')])
+        _spends.append(_sales)
+        column_names.append('NRPU')
+        scenario_df = pd.DataFrame(_spends).T
+        scenario_df.columns = column_names
+        ## write to sheet
+        ws = wb.create_sheet(scenario_name)
+        scenario_df_to_worksheet(scenario_df, ws)    
+        summary_rows.append(['Total',
+                        scenario_dict.get('modified_total_spends') ,
+                        scenario_dict.get('modified_total_sales'),
+                        scenario_dict.get('modified_total_sales') / scenario_dict.get('modified_total_spends'),
+                        '-',
+                        scenario_dict.get('modified_total_spends') / scenario_dict.get('modified_total_sales')])
+        columns_index = pd.MultiIndex.from_product([[''],['Channel']], names=["first", "second"])
+        columns_index = columns_index.append(pd.MultiIndex.from_product([[scenario_name],['Spends','NRPU','ROI','MROI','Spends per NRPU']], names=["first", "second"]))
+        if summary_df is None:
+            summary_df = pd.DataFrame(summary_rows, columns = columns_index)
+            summary_df = summary_df.set_index(('','Channel'))
+        else:
+            _df = pd.DataFrame(summary_rows, columns = columns_index)
+            _df = _df.set_index(('','Channel'))
+            summary_df = summary_df.merge(_df, left_index=True, right_index=True)
+    ws = wb.create_sheet('Summary',0)
+    summary_df_to_worksheet(summary_df.reset_index(), ws)
+    wb.save(st.session_state['xlsx_buffer'])
+    st.session_state['disable_download_button'] = False
+
+def disable_download_button():
+    st.session_state['disable_download_button'] =True
+
+def transform(x):
+    if x.name == ("",'Channel'):
+        return x
+    elif x.name[0] == 'ROI' or x.name[0] == 'MROI':
+        return x.apply(lambda y : y if isinstance(y,str) else decimal_formater(format_numbers(y,include_indicator=False,n_decimals=4),n_decimals=4))
+    else:
+        return x.apply(lambda y : y if isinstance(y,str) else format_numbers(y))
+
+def delete_scenario():
+    if selected_scenario in st.session_state['saved_scenarios']:
+        del st.session_state['saved_scenarios'][selected_scenario]
+        with open('../saved_scenarios.pkl', 'wb') as f:
+            pickle.dump(st.session_state['saved_scenarios'],f)     
+            
+def load_scenario():
+    if selected_scenario in st.session_state['saved_scenarios']:
+        st.session_state['scenario'] = class_from_dict(selected_scenario_details)
+        
+
+
+authenticator = st.session_state.get('authenticator')
+if authenticator is None:
+    authenticator = load_authenticator()
+
+name, authentication_status, username = authenticator.login('Login', 'main')
+auth_status = st.session_state.get('authentication_status')
+
+if auth_status == True:
+    is_state_initiaized = st.session_state.get('initialized',False)
+    if not is_state_initiaized:
+        #print("Scenario page state reloaded")
+        initialize_data()
+
+
+    saved_scenarios = st.session_state['saved_scenarios']
+
+
+    if len(saved_scenarios) ==0:
+        st.header('No saved scenarios')
+        
+    else:
+        
+        with st.sidebar:
+            selected_scenario = st.radio(
+                'Pick a scenario to view details',
+                list(saved_scenarios.keys())
+            )
+            st.markdown("""<hr>""", unsafe_allow_html=True)
+            scenarios_to_download = st.multiselect('Select scenarios to download',
+                        list(saved_scenarios.keys()))
+            
+            st.button('Prepare download',on_click=download_scenarios)
+            st.download_button(
+                    label="Download Scenarios",
+                    data=st.session_state['xlsx_buffer'].getvalue(),
+                    file_name="scenarios.xlsx",
+                    mime="application/vnd.ms-excel",
+                    disabled= st.session_state['disable_download_button'],
+                    on_click= disable_download_button
+                )
+            
+        column_1, column_2,column_3 = st.columns((6,1,1))
+        with column_1:
+            st.header(selected_scenario)
+        with column_2:
+            st.button('Delete scenarios', on_click=delete_scenario)
+        with column_3:
+            st.button('Load Scenario', on_click=load_scenario)
+        
+        selected_scenario_details = saved_scenarios[selected_scenario]
+        
+        pd.set_option('display.max_colwidth', 100)
+            
+        st.markdown(create_scenario_summary(selected_scenario_details).transform(transform).style.set_table_styles(
+    [{
+        'selector': 'th',
+        'props': [('background-color', '#11B6BD')]
+    },
+        {
+        'selector' : 'tr:nth-child(even)',
+        'props' : [('background-color', '#11B6BD')]
+        }
+        ]).to_html(),unsafe_allow_html=True)
+        
+elif auth_status == False:
+    st.error('Username/Password is incorrect')
+    
+if auth_status != True:
+    try:
+        username_forgot_pw, email_forgot_password, random_password = authenticator.forgot_password('Forgot password')
+        if username_forgot_pw:
+            st.success('New password sent securely')
+            # Random password to be transferred to user securely
+        elif username_forgot_pw == False:
+            st.error('Username not found')
+    except Exception as e:
+        st.error(e)

pages/Data_Import.py

@@ -0,0 +1,891 @@
+# Importing necessary libraries
+import streamlit as st
+
+st.set_page_config(
+    page_title="Model Build",
+    page_icon=":shark:",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+import numpy as np
+import pandas as pd
+from utilities import set_header, load_local_css, load_authenticator
+import pickle
+
+
+load_local_css("styles.css")
+set_header()
+
+authenticator = st.session_state.get("authenticator")
+if authenticator is None:
+    authenticator = load_authenticator()
+
+name, authentication_status, username = authenticator.login("Login", "main")
+auth_status = st.session_state.get("authentication_status")
+
+# Check for authentication status
+if auth_status != True:
+    st.stop()
+
+
+# Function to validate date column in dataframe
+def validate_date_column(df):
+    try:
+        # Attempt to convert the 'Date' column to datetime
+        df["date"] = pd.to_datetime(df["date"], format="%d-%m-%Y")
+        return True
+    except:
+        return False
+
+
+# Function to determine data interval
+def determine_data_interval(common_freq):
+    if common_freq == 1:
+        return "daily"
+    elif common_freq == 7:
+        return "weekly"
+    elif 28 <= common_freq <= 31:
+        return "monthly"
+    else:
+        return "irregular"
+
+
+# Function to read each uploaded Excel file into a pandas DataFrame and stores them in a dictionary
+st.cache_resource(show_spinner=False)
+
+
+def files_to_dataframes(uploaded_files):
+    df_dict = {}
+    for uploaded_file in uploaded_files:
+        # Extract file name without extension
+        file_name = uploaded_file.name.rsplit(".", 1)[0]
+
+        # Check for duplicate file names
+        if file_name in df_dict:
+            st.warning(
+                f"Duplicate File: {file_name}. This file will be skipped.",
+                icon="⚠️",
+            )
+            continue
+
+        # Read the file into a DataFrame
+        df = pd.read_excel(uploaded_file)
+
+        # Convert all column names to lowercase
+        df.columns = df.columns.str.lower().str.strip()
+
+        # Separate numeric and non-numeric columns
+        numeric_cols = list(df.select_dtypes(include=["number"]).columns)
+        non_numeric_cols = [
+            col
+            for col in df.select_dtypes(exclude=["number"]).columns
+            if col.lower() != "date"
+        ]
+
+        # Check for 'Date' column
+        if not (validate_date_column(df) and len(numeric_cols) > 0):
+            st.warning(
+                f"File Name: {file_name} ➜ Please upload data with Date column in 'DD-MM-YYYY' format and at least one media/exogenous column. This file will be skipped.",
+                icon="⚠️",
+            )
+            continue
+
+        # Check for interval
+        common_freq = common_freq = (
+            pd.Series(df["date"].unique()).diff().dt.days.dropna().mode()[0]
+        )
+        # Calculate the data interval (daily, weekly, monthly or irregular)
+        interval = determine_data_interval(common_freq)
+        if interval == "irregular":
+            st.warning(
+                f"File Name: {file_name} ➜ Please upload data in daily, weekly or monthly interval. This file will be skipped.",
+                icon="⚠️",
+            )
+            continue
+
+        # Store both DataFrames in the dictionary under their respective keys
+        df_dict[file_name] = {
+            "numeric": numeric_cols,
+            "non_numeric": non_numeric_cols,
+            "interval": interval,
+            "df": df,
+        }
+
+    return df_dict
+
+
+# Function to adjust dataframe granularity
+# def adjust_dataframe_granularity(df, current_granularity, target_granularity):
+#     # Set index
+#     df.set_index("date", inplace=True)
+
+#     # Define aggregation rules for resampling
+#     aggregation_rules = {
+#         col: "sum" if pd.api.types.is_numeric_dtype(df[col]) else "first"
+#         for col in df.columns
+#     }
+
+#     resampled_df = df
+#     if current_granularity == "daily" and target_granularity == "weekly":
+#         resampled_df = df.resample("W-MON").agg(aggregation_rules)
+
+#     elif current_granularity == "daily" and target_granularity == "monthly":
+#         resampled_df = df.resample("MS").agg(aggregation_rules)
+
+#     elif current_granularity == "daily" and target_granularity == "daily":
+#         resampled_df = df.resample("D").agg(aggregation_rules)
+
+#     elif current_granularity in ["weekly", "monthly"] and target_granularity == "daily":
+#         # For higher to lower granularity, distribute numeric and replicate non-numeric values equally across the new period
+#         expanded_data = []
+#         for _, row in df.iterrows():
+#             if current_granularity == "weekly":
+#                 period_range = pd.date_range(start=row.name, periods=7)
+#             elif current_granularity == "monthly":
+#                 period_range = pd.date_range(
+#                     start=row.name, periods=row.name.days_in_month
+#                 )
+
+#             for date in period_range:
+#                 new_row = {}
+#                 for col in df.columns:
+#                     if pd.api.types.is_numeric_dtype(df[col]):
+#                         if current_granularity == "weekly":
+#                             new_row[col] = row[col] / 7
+#                         elif current_granularity == "monthly":
+#                             new_row[col] = row[col] / row.name.days_in_month
+#                     else:
+#                         new_row[col] = row[col]
+#                 expanded_data.append((date, new_row))
+
+#         resampled_df = pd.DataFrame(
+#             [data for _, data in expanded_data],
+#             index=[date for date, _ in expanded_data],
+#         )
+
+#     # Reset index
+#     resampled_df = resampled_df.reset_index().rename(columns={"index": "date"})
+
+#     return resampled_df
+
+
+def adjust_dataframe_granularity(df, current_granularity, target_granularity):
+    # Set index
+    df.set_index("date", inplace=True)
+
+    # Define aggregation rules for resampling
+    aggregation_rules = {
+        col: "sum" if pd.api.types.is_numeric_dtype(df[col]) else "first"
+        for col in df.columns
+    }
+
+    # Initialize resampled_df
+    resampled_df = df
+    if current_granularity == "daily" and target_granularity == "weekly":
+        resampled_df = df.resample("W-MON", closed="left", label="left").agg(
+            aggregation_rules
+        )
+
+    elif current_granularity == "daily" and target_granularity == "monthly":
+        resampled_df = df.resample("MS", closed="left", label="left").agg(
+            aggregation_rules
+        )
+
+    elif current_granularity == "daily" and target_granularity == "daily":
+        resampled_df = df.resample("D").agg(aggregation_rules)
+
+    elif current_granularity in ["weekly", "monthly"] and target_granularity == "daily":
+        # For higher to lower granularity, distribute numeric and replicate non-numeric values equally across the new period
+        expanded_data = []
+        for _, row in df.iterrows():
+            if current_granularity == "weekly":
+                period_range = pd.date_range(start=row.name, periods=7)
+            elif current_granularity == "monthly":
+                period_range = pd.date_range(
+                    start=row.name, periods=row.name.days_in_month
+                )
+
+            for date in period_range:
+                new_row = {}
+                for col in df.columns:
+                    if pd.api.types.is_numeric_dtype(df[col]):
+                        if current_granularity == "weekly":
+                            new_row[col] = row[col] / 7
+                        elif current_granularity == "monthly":
+                            new_row[col] = row[col] / row.name.days_in_month
+                    else:
+                        new_row[col] = row[col]
+                expanded_data.append((date, new_row))
+
+        resampled_df = pd.DataFrame(
+            [data for _, data in expanded_data],
+            index=[date for date, _ in expanded_data],
+        )
+
+    # Reset index
+    resampled_df = resampled_df.reset_index().rename(columns={"index": "date"})
+
+    return resampled_df
+
+
+# Function to clean and extract unique values of DMA and Panel
+st.cache_resource(show_spinner=False)
+
+
+def clean_and_extract_unique_values(files_dict, selections):
+    all_dma_values = set()
+    all_panel_values = set()
+
+    for file_name, file_data in files_dict.items():
+        df = file_data["df"]
+
+        # 'DMA' and 'Panel' selections
+        selected_dma = selections[file_name].get("DMA")
+        selected_panel = selections[file_name].get("Panel")
+
+        # Clean and standardize DMA column if it exists and is selected
+        if selected_dma and selected_dma != "N/A" and selected_dma in df.columns:
+            df[selected_dma] = (
+                df[selected_dma].str.lower().str.strip().str.replace("_", " ")
+            )
+            all_dma_values.update(df[selected_dma].dropna().unique())
+
+        # Clean and standardize Panel column if it exists and is selected
+        if selected_panel and selected_panel != "N/A" and selected_panel in df.columns:
+            df[selected_panel] = (
+                df[selected_panel].str.lower().str.strip().str.replace("_", " ")
+            )
+            all_panel_values.update(df[selected_panel].dropna().unique())
+
+        # Update the processed DataFrame back in the dictionary
+        files_dict[file_name]["df"] = df
+
+    return all_dma_values, all_panel_values
+
+
+# Function to format values for display
+st.cache_resource(show_spinner=False)
+
+
+def format_values_for_display(values_list):
+    # Capitalize the first letter of each word and replace underscores with spaces
+    formatted_list = [value.replace("_", " ").title() for value in values_list]
+    # Join values with commas and 'and' before the last value
+    if len(formatted_list) > 1:
+        return ", ".join(formatted_list[:-1]) + ", and " + formatted_list[-1]
+    elif formatted_list:
+        return formatted_list[0]
+    return "No values available"
+
+
+# Function to normalizes all data within files_dict to a daily granularity
+st.cache(show_spinner=False, allow_output_mutation=True)
+
+
+def standardize_data_to_daily(files_dict, selections):
+    # Normalize all data to a daily granularity using a provided function
+    files_dict = apply_granularity_to_all(files_dict, "daily", selections)
+
+    # Update the "interval" attribute for each dataset to indicate the new granularity
+    for files_name, files_data in files_dict.items():
+        files_data["interval"] = "daily"
+
+    return files_dict
+
+
+# Function to apply granularity transformation to all DataFrames in files_dict
+st.cache_resource(show_spinner=False)
+
+
+def apply_granularity_to_all(files_dict, granularity_selection, selections):
+    for file_name, file_data in files_dict.items():
+        df = file_data["df"].copy()
+
+        # Handling when DMA or Panel might be 'N/A'
+        selected_dma = selections[file_name].get("DMA")
+        selected_panel = selections[file_name].get("Panel")
+
+        # Correcting the segment selection logic & handling 'N/A'
+        if selected_dma != "N/A" and selected_panel != "N/A":
+            unique_combinations = df[[selected_dma, selected_panel]].drop_duplicates()
+        elif selected_dma != "N/A":
+            unique_combinations = df[[selected_dma]].drop_duplicates()
+            selected_panel = None  # Ensure Panel is ignored if N/A
+        elif selected_panel != "N/A":
+            unique_combinations = df[[selected_panel]].drop_duplicates()
+            selected_dma = None  # Ensure DMA is ignored if N/A
+        else:
+            # If both are 'N/A', process the entire dataframe as is
+            df = adjust_dataframe_granularity(
+                df, file_data["interval"], granularity_selection
+            )
+            files_dict[file_name]["df"] = df
+            continue  # Skip to the next file
+
+        transformed_segments = []
+        for _, combo in unique_combinations.iterrows():
+            if selected_dma and selected_panel:
+                segment = df[
+                    (df[selected_dma] == combo[selected_dma])
+                    & (df[selected_panel] == combo[selected_panel])
+                ]
+            elif selected_dma:
+                segment = df[df[selected_dma] == combo[selected_dma]]
+            elif selected_panel:
+                segment = df[df[selected_panel] == combo[selected_panel]]
+
+            # Adjust granularity of the segment
+            transformed_segment = adjust_dataframe_granularity(
+                segment, file_data["interval"], granularity_selection
+            )
+            transformed_segments.append(transformed_segment)
+
+        # Combine all transformed segments into a single DataFrame for this file
+        transformed_df = pd.concat(transformed_segments, ignore_index=True)
+        files_dict[file_name]["df"] = transformed_df
+
+    return files_dict
+
+
+# Function to create main dataframe structure
+st.cache_resource(show_spinner=False)
+
+
+def create_main_dataframe(
+    files_dict, all_dma_values, all_panel_values, granularity_selection
+):
+    # Determine the global start and end dates across all DataFrames
+    global_start = min(df["df"]["date"].min() for df in files_dict.values())
+    global_end = max(df["df"]["date"].max() for df in files_dict.values())
+
+    # Adjust the date_range generation based on the granularity_selection
+    if granularity_selection == "weekly":
+        # Generate a weekly range, with weeks starting on Monday
+        date_range = pd.date_range(start=global_start, end=global_end, freq="W-MON")
+    elif granularity_selection == "monthly":
+        # Generate a monthly range, starting from the first day of each month
+        date_range = pd.date_range(start=global_start, end=global_end, freq="MS")
+    else:  # Default to daily if not weekly or monthly
+        date_range = pd.date_range(start=global_start, end=global_end, freq="D")
+
+    # Collect all unique DMA and Panel values, excluding 'N/A'
+    all_dmas = all_dma_values
+    all_panels = all_panel_values
+
+    # Dynamically build the list of dimensions (Panel, DMA) to include in the main DataFrame based on availability
+    dimensions, merge_keys = [], []
+    if all_panels:
+        dimensions.append(all_panels)
+        merge_keys.append("Panel")
+    if all_dmas:
+        dimensions.append(all_dmas)
+        merge_keys.append("DMA")
+
+    dimensions.append(date_range)  # Date range is always included
+    merge_keys.append("date")  # Date range is always included
+
+    # Create a main DataFrame template with the dimensions
+    main_df = pd.MultiIndex.from_product(
+        dimensions,
+        names=[name for name, _ in zip(merge_keys, dimensions)],
+    ).to_frame(index=False)
+
+    return main_df.reset_index(drop=True)
+
+
+# Function to prepare and merge dataFrames
+st.cache_resource(show_spinner=False)
+
+
+def merge_into_main_df(main_df, files_dict, selections):
+    for file_name, file_data in files_dict.items():
+        df = file_data["df"].copy()
+
+        # Rename selected DMA and Panel columns if not 'N/A'
+        selected_dma = selections[file_name].get("DMA", "N/A")
+        selected_panel = selections[file_name].get("Panel", "N/A")
+        if selected_dma != "N/A":
+            df.rename(columns={selected_dma: "DMA"}, inplace=True)
+        if selected_panel != "N/A":
+            df.rename(columns={selected_panel: "Panel"}, inplace=True)
+
+        # Merge current DataFrame into main_df based on 'date', and where applicable, 'Panel' and 'DMA'
+        merge_keys = ["date"]
+        if "Panel" in df.columns:
+            merge_keys.append("Panel")
+        if "DMA" in df.columns:
+            merge_keys.append("DMA")
+        main_df = pd.merge(main_df, df, on=merge_keys, how="left")
+
+    # After all merges, sort by 'date' and reset index for cleanliness
+    sort_by = ["date"]
+    if "Panel" in main_df.columns:
+        sort_by.append("Panel")
+    if "DMA" in main_df.columns:
+        sort_by.append("DMA")
+    main_df.sort_values(by=sort_by, inplace=True)
+    main_df.reset_index(drop=True, inplace=True)
+
+    return main_df
+
+
+# Function to categorize column
+def categorize_column(column_name):
+    # Define keywords for each category
+    internal_keywords = [
+        "Price",
+        "Discount",
+        "product_price",
+        "cost",
+        "margin",
+        "inventory",
+        "sales",
+        "revenue",
+        "turnover",
+        "expense",
+    ]
+    exogenous_keywords = [
+        "GDP",
+        "Tax",
+        "Inflation",
+        "interest_rate",
+        "employment_rate",
+        "exchange_rate",
+        "consumer_spending",
+        "retail_sales",
+        "oil_prices",
+        "weather",
+    ]
+
+    # Check if the column name matches any of the keywords for Internal or Exogenous categories
+    for keyword in internal_keywords:
+        if keyword.lower() in column_name.lower():
+            return "Internal"
+    for keyword in exogenous_keywords:
+        if keyword.lower() in column_name.lower():
+            return "Exogenous"
+
+    # Default to Media if no match found
+    return "Media"
+
+
+# Function to calculate missing stats and prepare for editable DataFrame
+st.cache_resource(show_spinner=False)
+
+
+def prepare_missing_stats_df(df):
+    missing_stats = []
+    for column in df.columns:
+        if (
+            column == "date" or column == "DMA" or column == "Panel"
+        ):  # Skip Date, DMA and Panel column
+            continue
+
+        missing = df[column].isnull().sum()
+        pct_missing = round((missing / len(df)) * 100, 2)
+
+        # Dynamically assign category based on column name
+        # category = categorize_column(column)
+        category = "Media"
+
+        missing_stats.append(
+            {
+                "Column": column,
+                "Missing Values": missing,
+                "Missing Percentage": pct_missing,
+                "Impute Method": "Fill with 0",  # Default value
+                "Category": category,
+            }
+        )
+    stats_df = pd.DataFrame(missing_stats)
+
+    return stats_df
+
+
+# Function to add API DataFrame details to the files dictionary
+st.cache_resource(show_spinner=False)
+
+
+def add_api_dataframe_to_dict(main_df, files_dict):
+    files_dict["API"] = {
+        "numeric": list(main_df.select_dtypes(include=["number"]).columns),
+        "non_numeric": [
+            col
+            for col in main_df.select_dtypes(exclude=["number"]).columns
+            if col.lower() != "date"
+        ],
+        "interval": determine_data_interval(
+            pd.Series(main_df["date"].unique()).diff().dt.days.dropna().mode()[0]
+        ),
+        "df": main_df,
+    }
+
+    return files_dict
+
+
+# Function to reads an API into a DataFrame, parsing specified columns as datetime
+@st.cache_resource(show_spinner=False)
+def read_API_data():
+    return pd.read_excel(r"upf_data_converted.xlsx", parse_dates=["Date"])
+
+
+# Function to set the 'DMA_Panel_Selected' session state variable to False
+def set_DMA_Panel_Selected_false():
+    st.session_state["DMA_Panel_Selected"] = False
+
+
+# Initialize 'final_df' in session state
+if "final_df" not in st.session_state:
+    st.session_state["final_df"] = pd.DataFrame()
+
+# Initialize 'bin_dict' in session state
+if "bin_dict" not in st.session_state:
+    st.session_state["bin_dict"] = {}
+
+# Initialize 'DMA_Panel_Selected' in session state
+if "DMA_Panel_Selected" not in st.session_state:
+    st.session_state["DMA_Panel_Selected"] = False
+
+# Page Title
+st.write("")  # Top padding
+st.title("Data Import")
+
+
+#########################################################################################################################################################
+# Create a dictionary to hold all DataFrames and collect user input to specify "DMA" and "Panel" columns for each file
+#########################################################################################################################################################
+
+
+# Read the Excel file, parsing 'Date' column as datetime
+main_df = read_API_data()
+
+# Convert all column names to lowercase
+main_df.columns = main_df.columns.str.lower().str.strip()
+
+# File uploader
+uploaded_files = st.file_uploader(
+    "Upload additional data",
+    type=["xlsx"],
+    accept_multiple_files=True,
+    on_change=set_DMA_Panel_Selected_false,
+)
+
+# Custom HTML for upload instructions
+recommendation_html = f"""
+<div style="text-align: justify;">
+<strong>Recommendation:</strong> For optimal processing, please ensure that all uploaded datasets including DMA, Panel, media, internal, and exogenous data adhere to the following guidelines: Each dataset must include a <code>Date</code> column formatted as <code>DD-MM-YYYY</code>, be free of missing values.
+</div>
+"""
+st.markdown(recommendation_html, unsafe_allow_html=True)
+
+# Choose Date Granularity
+st.markdown("#### Choose Date Granularity")
+# Granularity Selection
+granularity_selection = st.selectbox(
+    "Choose Date Granularity",
+    ["Daily", "Weekly", "Monthly"],
+    label_visibility="collapsed",
+    on_change=set_DMA_Panel_Selected_false,
+)
+granularity_selection = str(granularity_selection).lower()
+
+# Convert files to dataframes
+files_dict = files_to_dataframes(uploaded_files)
+
+# Add API Dataframe
+if main_df is not None:
+    files_dict = add_api_dataframe_to_dict(main_df, files_dict)
+
+# Display a warning message if no files have been uploaded and halt further execution
+if not files_dict:
+    st.warning(
+        "Please upload at least one file to proceed.",
+        icon="⚠️",
+    )
+    st.stop()  # Halts further execution until file is uploaded
+
+
+# Select DMA and Panel columns
+st.markdown("#### Select DMA and Panel columns")
+selections = {}
+with st.expander("Select DMA and Panel columns", expanded=False):
+    count = 0  # Initialize counter to manage the visibility of labels and keys
+    for file_name, file_data in files_dict.items():
+        # Determine visibility of the label based on the count
+        if count == 0:
+            label_visibility = "visible"
+        else:
+            label_visibility = "collapsed"
+
+        # Extract non-numeric columns
+        non_numeric_cols = file_data["non_numeric"]
+
+        # Prepare DMA and Panel values for dropdown, adding "N/A" as an option
+        dma_values = non_numeric_cols + ["N/A"]
+        panel_values = non_numeric_cols + ["N/A"]
+
+        # Skip if only one option is available
+        if len(dma_values) == 1 and len(panel_values) == 1:
+            selected_dma, selected_panel = "N/A", "N/A"
+            # Update the selections for DMA and Panel for the current file
+            selections[file_name] = {
+                "DMA": selected_dma,
+                "Panel": selected_panel,
+            }
+            continue
+
+        # Create layout columns for File Name, DMA, and Panel selections
+        file_name_col, DMA_col, Panel_col = st.columns([2, 4, 4])
+
+        with file_name_col:
+            # Display "File Name" label only for the first file
+            if count == 0:
+                st.write("File Name")
+            else:
+                st.write("")
+            st.write(file_name)  # Display the file name
+
+        with DMA_col:
+            # Display a selectbox for DMA values
+            selected_dma = st.selectbox(
+                "Select DMA",
+                dma_values,
+                on_change=set_DMA_Panel_Selected_false,
+                label_visibility=label_visibility,  # Control visibility of the label
+                key=f"DMA_selectbox{count}",  # Ensure unique key for each selectbox
+            )
+
+        with Panel_col:
+            # Display a selectbox for Panel values
+            selected_panel = st.selectbox(
+                "Select Panel",
+                panel_values,
+                on_change=set_DMA_Panel_Selected_false,
+                label_visibility=label_visibility,  # Control visibility of the label
+                key=f"Panel_selectbox{count}",  # Ensure unique key for each selectbox
+            )
+
+        # Skip processing if the same column is selected for both Panel and DMA due to potential data integrity issues
+        if selected_panel == selected_dma and not (
+            selected_panel == "N/A" and selected_dma == "N/A"
+        ):
+            st.warning(
+                f"File: {file_name} → The same column cannot serve as both Panel and DMA. Please adjust your selections.",
+            )
+            selected_dma, selected_panel = "N/A", "N/A"
+            st.stop()
+
+        # Update the selections for DMA and Panel for the current file
+        selections[file_name] = {
+            "DMA": selected_dma,
+            "Panel": selected_panel,
+        }
+
+        count += 1  # Increment the counter after processing each file
+
+    # Accept DMA and Panel selection
+    if st.button("Accept and Process", use_container_width=True):
+
+        # Normalize all data to a daily granularity. This initial standardization simplifies subsequent conversions to other levels of granularity
+        with st.spinner("Processing...", cache=True):
+            files_dict = standardize_data_to_daily(files_dict, selections)
+
+            # Convert all data to daily level granularity
+            files_dict = apply_granularity_to_all(
+                files_dict, granularity_selection, selections
+            )
+
+        st.session_state["files_dict"] = files_dict
+        st.session_state["DMA_Panel_Selected"] = True
+
+
+#########################################################################################################################################################
+# Display unique DMA and Panel values
+#########################################################################################################################################################
+
+
+# Halts further execution until DMA and Panel columns are selected
+if "files_dict" in st.session_state and st.session_state["DMA_Panel_Selected"]:
+    files_dict = st.session_state["files_dict"]
+else:
+    st.stop()
+
+# Set to store unique values of DMA and Panel
+with st.spinner("Fetching DMA and Panel values..."):
+    all_dma_values, all_panel_values = clean_and_extract_unique_values(
+        files_dict, selections
+    )
+
+    # List of DMA and Panel columns unique values
+    list_of_all_dma_values = list(all_dma_values)
+    list_of_all_panel_values = list(all_panel_values)
+
+    # Format DMA and Panel values for display
+    formatted_dma_values = format_values_for_display(list_of_all_dma_values)
+    formatted_panel_values = format_values_for_display(list_of_all_panel_values)
+
+# Unique DMA and Panel values
+st.markdown("#### Unique DMA and Panel values")
+# Display DMA and Panel values
+with st.expander("Unique DMA and Panel values"):
+    st.write("")
+    st.markdown(
+        f"""
+    <style>
+    .justify-text {{
+    text-align: justify;
+    }}
+    </style>
+    <div class="justify-text">
+    <strong>Panel Values:</strong> {formatted_panel_values}<br>
+    <strong>DMA Values:</strong> {formatted_dma_values}
+    </div>
+    """,
+        unsafe_allow_html=True,
+    )
+
+    # Display total DMA and Panel
+    st.write("")
+    st.markdown(
+        f"""
+    <div style="text-align: justify;">
+        <strong>Number of DMAs detected:</strong> {len(list_of_all_dma_values)}<br>
+        <strong>Number of Panels detected:</strong> {len(list_of_all_panel_values)}
+    </div>
+    """,
+        unsafe_allow_html=True,
+    )
+    st.write("")
+
+
+#########################################################################################################################################################
+# Merge all DataFrames
+#########################################################################################################################################################
+
+
+# Merge all DataFrames selected
+main_df = create_main_dataframe(
+    files_dict, all_dma_values, all_panel_values, granularity_selection
+)
+merged_df = merge_into_main_df(main_df, files_dict, selections)
+
+# # Display the merged DataFrame
+# st.markdown("#### Merged DataFrame based on selected DMA and Panel")
+# st.dataframe(merged_df)
+
+
+#########################################################################################################################################################
+# Categorize Variables and Impute Missing Values
+#########################################################################################################################################################
+
+
+# Create an editable DataFrame in Streamlit
+st.markdown("#### Select Variables Category & Impute Missing Values")
+
+# Prepare missing stats DataFrame for editing
+missing_stats_df = prepare_missing_stats_df(merged_df)
+
+edited_stats_df = st.data_editor(
+    missing_stats_df,
+    column_config={
+        "Impute Method": st.column_config.SelectboxColumn(
+            options=[
+                "Drop Column",
+                "Fill with Mean",
+                "Fill with Median",
+                "Fill with 0",
+            ],
+            required=True,
+            default="Fill with 0",
+        ),
+        "Category": st.column_config.SelectboxColumn(
+            options=[
+                "Media",
+                "Exogenous",
+                "Internal",
+                "Response_Metric"
+            ],
+            required=True,
+            default="Media",
+        ),
+    },
+    disabled=["Column", "Missing Values", "Missing Percentage"],
+    hide_index=True,
+    use_container_width=True,
+)
+
+# Apply changes based on edited DataFrame
+for i, row in edited_stats_df.iterrows():
+    column = row["Column"]
+    if row["Impute Method"] == "Drop Column":
+        merged_df.drop(columns=[column], inplace=True)
+
+    elif row["Impute Method"] == "Fill with Mean":
+        merged_df[column].fillna(merged_df[column].mean(), inplace=True)
+
+    elif row["Impute Method"] == "Fill with Median":
+        merged_df[column].fillna(merged_df[column].median(), inplace=True)
+
+    elif row["Impute Method"] == "Fill with 0":
+        merged_df[column].fillna(0, inplace=True)
+
+# Display the Final DataFrame and exogenous variables
+st.markdown("#### Final DataFrame")
+final_df = merged_df
+st.dataframe(final_df, hide_index=True)
+
+# Initialize an empty dictionary to hold categories and their variables
+category_dict = {}
+
+# Iterate over each row in the edited DataFrame to populate the dictionary
+for i, row in edited_stats_df.iterrows():
+    column = row["Column"]
+    category = row["Category"]  # The category chosen by the user for this variable
+
+    # Check if the category already exists in the dictionary
+    if category not in category_dict:
+        # If not, initialize it with the current column as its first element
+        category_dict[category] = [column]
+    else:
+        # If it exists, append the current column to the list of variables under this category
+        category_dict[category].append(column)
+
+# Add Date, DMA and Panel in category dictionary
+category_dict.update({"Date": ["date"]})
+if "DMA" in final_df.columns:
+    category_dict["DMA"] = ["DMA"]
+
+if "Panel" in final_df.columns:
+    category_dict["Panel"] = ["Panel"]
+
+# Display the dictionary
+st.markdown("#### Variable Category")
+for category, variables in category_dict.items():
+    # Check if there are multiple variables to handle "and" insertion correctly
+    if len(variables) > 1:
+        # Join all but the last variable with ", ", then add " and " before the last variable
+        variables_str = ", ".join(variables[:-1]) + " and " + variables[-1]
+    else:
+        # If there's only one variable, no need for "and"
+        variables_str = variables[0]
+
+    # Display the category and its variables in the desired format
+    st.markdown(
+        f"<div style='text-align: justify;'><strong>{category}:</strong> {variables_str}</div>",
+        unsafe_allow_html=True,
+    )
+
+# Store final dataframe and bin dictionary into session state
+st.session_state["final_df"], st.session_state["bin_dict"] = final_df, category_dict
+
+if st.button('Save Changes'):
+    
+    with open("Pickle_files/main_df", 'wb') as f:
+        pickle.dump(st.session_state["final_df"], f)
+    with open("Pickle_files/category_dict",'wb') as c:
+        pickle.dump(st.session_state["bin_dict"],c)
+    st.success('Changes Saved!')
+
+
+

pages/actual_data.csv

@@ -0,0 +1,158 @@
+const,clicks_search_decay.2,impressions_tv_lag3,online_edu_trend_lag3,clicks_digital_lag2_decay.3,impressions_streaming_lag2_decay.4,covid_cases_lag3,unemployement_rate_lead4,season,flag_Aug_1,flag_Aug_2,flag_Aug_3,flag_dec_1,flag_dec_-1,flag_dec_-2,flag_dec_-3,flag_easter_-1,flag_easter_-2,flag_may_-1,flag_may_-2,flag_jun_-1,flag_jun_-2,covid_flag1,flag_june28,flag_aug13,flag_sep13,flag_mar_feb,date,total_prospect_id
+1.0,0.03264506089026503,0.0,0.0,0.0,0.11920857922376585,0.0,0.2448979591836735,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-11-10,3106
+1.0,0.1203178311529351,0.0,0.0,0.0,0.23575959332216032,0.0,0.2448979591836735,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-11-17,7809
+1.0,0.037674240888288246,0.0,0.0,0.30427286753070926,0.14866425214344534,0.0,0.2448979591836735,102,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-11-24,5658
+1.0,0.114056065999327,0.25459834519940233,0.5700000000000001,0.3210660307498862,0.06375317695001911,0.0,0.2448979591836735,103,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-01,7528
+1.0,0.15091848146432302,0.04759636387261456,0.58,0.2652143429433443,0.02550166207848893,0.0,0.2380952380952381,104,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-08,8913
+1.0,0.09691798534505919,0.0,0.41000000000000003,0.27398476053158455,0.22803554179688423,0.0,0.2380952380952381,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-15,7974
+1.0,0.0,0.2185391903071715,0.53,0.3093665823461814,0.3016670242357716,0.0,0.2380952380952381,106,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-22,5034
+1.0,0.06818143419410627,0.0645557652165116,0.6,0.35005256364095544,0.3915886857834677,0.0,0.2380952380952381,107,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-29,8296
+1.0,0.19748095587743647,0.0,0.49,0.2866388037412839,0.4644891817948484,0.0,0.2380952380952381,108,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-01-05,10953
+1.0,0.2718903484441833,0.31632836028874944,0.42,0.38339772931601046,0.4758788391710054,0.0,0.2380952380952381,109,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2020-01-12,11583
+1.0,0.29329394272923165,0.710207473795361,0.56,0.4716341482535363,0.47415700741999534,0.0,0.2380952380952381,110,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2020-01-19,11650
+1.0,0.3150710926081645,0.6225458397661645,0.66,0.5560651882029227,0.2282082561307921,0.0,0.2380952380952381,111,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-01-26,10086
+1.0,0.23335326208386092,0.5093471390869946,0.65,0.5990392189890996,0.09128427138188955,0.0,0.2993197278911565,112,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-02,8454
+1.0,0.18339704064539092,0.46920681970876166,0.66,0.5097387360461574,0.03651393215188798,0.0,0.2993197278911565,113,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-09,7842
+1.0,0.1829206162885479,0.5702922924005152,0.64,0.3647117781342298,0.5333315970976881,0.0,0.2993197278911565,114,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-16,8528
+1.0,0.17708137647064887,0.4762803199026322,0.62,0.2994390381863003,0.9999999999999999,0.0,0.2993197278911565,115,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-23,9230
+1.0,0.2110785179466496,0.31643298954206356,0.65,0.318727924805625,0.5153399788387041,0.0,0.2993197278911565,116,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-03-01,8210
+1.0,0.1922309642774856,0.35110354589746834,0.65,0.3435805763353255,0.20613623376787482,0.0,1.0,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-03-08,6573
+1.0,0.1174971533357681,0.4397302099507956,0.64,0.37079693119819457,0.08245451214041095,0.0,1.0,118,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2020-03-15,4464
+1.0,0.04487177585471158,0.5651604986093057,0.66,0.3797815418753292,0.032981804856164386,3.6661729553753427e-06,1.0,119,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2020-03-22,5498
+1.0,0.04417426781579725,0.5142518574426083,0.77,0.3239901926717436,0.013192796475509808,0.00016497778299189042,1.0,120,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-03-29,7134
+1.0,0.09508966430933447,0.4246084040047787,1.0,0.22766051203571303,0.005277118590203924,0.01074555293220513,0.8979591836734694,121,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,2020-04-05,6507
+1.0,0.1727148072921107,0.3306303340730278,0.92,0.2557126494916798,0.0021108474360815696,0.07506489126131015,0.8979591836734694,122,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,2020-04-12,6752
+1.0,0.2757761792524949,0.9059477066272279,0.87,0.2910560761584964,0.0008443389744326279,0.11051311756683434,0.8979591836734694,123,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-04-19,7874
+1.0,0.46164669127102737,1.0,0.8200000000000001,0.29288325042575475,0.0003377355897730512,0.1323451775160945,0.8979591836734694,124,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-04-26,8706
+1.0,0.3631365926708698,0.8555262504044332,0.85,0.3143348639913703,0.00013509423590922048,0.12527679605813083,0.8979591836734694,125,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-03,9593
+1.0,0.3556269301486625,0.5998066602658987,0.8,0.3573452157072908,5.4838924587260594e-05,0.08418266340132861,0.7482993197278912,126,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-10,9554
+1.0,0.3898924329688705,0.31953123019194307,0.76,0.3492819601843694,0.08837696494340691,0.06699197841357364,0.7482993197278912,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-17,9461
+1.0,0.3270785638817633,0.5040802333471541,0.88,0.37224504100306005,0.12944061135952373,0.04806352744497074,0.7482993197278912,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-24,8347
+1.0,0.29596428185745655,0.6228739252579004,0.8300000000000001,0.3873711562094451,0.14079607140381442,0.028926104617911456,0.7482993197278912,129,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-31,7926
+1.0,0.23446621861142697,0.644779308361226,0.8,0.3519020717491842,0.15750706055823313,0.024482702995996537,0.6938775510204082,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-06-07,8606
+1.0,0.2202508917985891,0.726916988225644,0.71,0.32726146750928653,0.0797309833640819,0.022000703905207433,0.6938775510204082,131,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-06-14,7573
+1.0,0.18610614076735926,0.5963517592669729,0.73,0.31618831243754153,0.03501476889363339,0.015086301711369536,0.6938775510204082,132,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2020-06-21,6983
+1.0,0.1568177529621934,0.6764095796293655,0.75,0.2836099513597926,0.014005944823975384,0.011489786042146325,0.6938775510204082,133,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-06-28,6277
+1.0,0.22774801916471138,0.6466210070345804,0.72,0.25409997289933184,0.006272411362367827,0.00871449311492719,0.5714285714285715,134,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-05,7421
+1.0,0.24542124594101095,0.6580063264819511,0.73,0.2516667689694555,0.05947462601462651,0.008318546435746652,0.5714285714285715,135,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-12,7852
+1.0,0.24895270375190542,0.32749815383926373,0.68,0.2671053898526598,0.0888609058832765,0.008014254080450499,0.5714285714285715,136,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-19,7396
+1.0,0.16285259960994197,0.3666961464656464,0.78,0.26077100654286645,0.12420199588573878,0.008058248155915004,0.5714285714285715,137,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-26,7041
+1.0,0.16864346155569104,0.39341698388602436,0.84,0.25893225300958655,0.10423952696584138,0.00920209411799211,0.5714285714285715,138,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-02,7470
+1.0,0.22582910125625383,0.41507293852636135,0.8300000000000001,0.2528768986269057,0.08197739941078482,0.009315745479608745,0.5374149659863946,139,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-09,8725
+1.0,0.2778946696783185,0.7857143231388266,0.8,0.2772125371796957,0.07178679747906064,0.007237025413910927,0.5374149659863946,140,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-16,9657
+1.0,0.3062154076077969,0.434016630925742,0.87,0.33174759696083367,0.12078972986041582,0.006500124649880482,0.5374149659863946,141,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,2020-08-23,10000
+1.0,0.2851073700683267,0.4051792323256236,0.8200000000000001,0.3621387745268235,0.1539969659046611,0.006118842662521447,0.5374149659863946,142,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-30,8941
+1.0,0.25999778433367665,0.4113785668398346,0.77,0.3604714968693371,0.1462622685965232,0.006375474769397721,0.4693877551020409,143,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-09-06,8507
+1.0,0.2947500457787596,0.43576671635701947,0.74,0.3084711376902622,0.1030893445960345,0.0060051913009048115,0.4693877551020409,144,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2020-09-13,9887
+1.0,0.3239559328273078,0.40721834097732834,0.72,0.24061271129609485,0.08422768334333634,0.006456130574415978,0.4693877551020409,145,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-09-20,9627
+1.0,0.3189849597494306,0.4831656702512836,0.68,0.28577062852640756,0.054400116894051116,0.006401137980085348,0.4693877551020409,146,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-09-27,8735
+1.0,0.2930673557404469,0.5423730023996388,0.62,0.32330756771945346,0.02176006539088146,0.007566980979894707,0.45578231292517013,147,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-04,8138
+1.0,0.27381401410957934,0.48862464971809444,0.59,0.33668984325037016,0.008704026156352586,0.009172764734349107,0.45578231292517013,148,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-11,7966
+1.0,0.21658154029531146,0.5162854532967293,0.55,0.44481231480084876,0.003481610462541034,0.012223020633221393,0.45578231292517013,149,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-18,8109
+1.0,0.21772903332032795,0.47368257634991157,0.6,0.46141705479304307,0.0013926441850164136,0.013601501664442522,0.45578231292517013,150,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-25,7848
+1.0,0.16712357438522701,0.5132571164009214,0.5,0.38402389059771924,0.0005570576740065655,0.012915927321787332,0.45578231292517013,151,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-01,6516
+1.0,0.1814031347156822,0.5409537987241609,0.5,0.2968208337801042,0.00022282306960262618,0.013091903623645349,0.45578231292517013,152,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-08,7233
+1.0,0.16852532779394064,0.49490997931858044,0.5,0.22663075929954526,8.912922784105048e-05,0.014624363918992243,0.45578231292517013,153,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-15,7409
+1.0,0.10492104198879731,0.4086344123814518,0.41000000000000003,0.21669561761817938,3.565169113642019e-05,0.016127494830696133,0.45578231292517013,154,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-22,6232
+1.0,0.16920169406380464,0.45151008168804235,0.49,0.21833619946593313,1.4260676454568076e-05,0.024849320291534072,0.45578231292517013,155,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-29,8170
+1.0,0.1305885456099783,0.4543635808918873,0.47000000000000003,0.1596898931167178,5.704270581827231e-06,0.03519159419864792,0.435374149659864,156,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-06,7075
+1.0,0.1214984593864375,0.35070760971315756,0.4,0.15417676852356046,2.2817082327308923e-06,0.041732046751037526,0.435374149659864,157,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-13,7379
+1.0,0.057042007816384965,0.32470890321593604,0.47000000000000003,0.15442387578570832,9.126832930923571e-07,0.049892947749703036,0.435374149659864,158,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-20,5442
+1.0,0.12406882983279183,0.3135816516054531,0.45,0.1671308209739812,3.650733172369429e-07,0.0686930826648678,0.435374149659864,159,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-27,7735
+1.0,0.24786523070013738,0.3102913429236421,0.42,0.16347790840061424,1.4602932689477716e-07,0.0732574679943101,0.435374149659864,160,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-03,9754
+1.0,0.26083059672146286,0.2649240941306087,0.34,0.25327016920452516,5.841173075791087e-08,0.07444897420480709,0.4217687074829932,161,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-10,10641
+1.0,0.24028847292133387,0.6513962629200784,0.38,0.3773812732234543,2.3364692303164347e-08,0.08318546435746653,0.4217687074829932,162,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2021-01-17,10230
+1.0,0.31526302386797916,0.531674302460824,0.47000000000000003,0.3527386460097067,9.345876921265738e-09,0.10258685163731283,0.4217687074829932,163,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-24,10352
+1.0,0.2966293410018717,0.44836670500794606,0.47000000000000003,0.3711695518795665,3.738350768506295e-09,0.13234151134313912,0.4217687074829932,164,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-31,9216
+1.0,0.20088776123137192,0.3815806999416851,0.45,0.33580461662371014,1.4953403074025183e-09,0.12043744775703538,0.40816326530612246,165,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-07,8421
+1.0,0.173394454128539,0.343687050600215,0.48,0.3277941002786073,5.981361229610074e-10,0.11271648751301491,0.40816326530612246,166,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-14,9281
+1.0,0.1777198044422716,0.33051072402008147,0.5,0.31487397296804576,2.3925444918440296e-10,0.109699227170741,0.40816326530612246,167,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-21,8891
+1.0,0.1850269016675808,0.30627520154343757,0.46,0.3133091660972597,9.570177967376119e-11,0.08255854878209734,0.40816326530612246,168,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-28,8169
+1.0,0.2529549962208855,0.298123038215738,0.42,0.3358964981168952,3.828071186950448e-11,0.08351908609640568,0.40816326530612246,169,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-07,8724
+1.0,0.213028120324469,0.3267901551549544,0.44,0.3038053348505854,1.531228474780179e-11,0.07285052279626343,0.40816326530612246,170,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-14,8194
+1.0,0.16441430466323353,0.25967469209260036,0.5,0.32087357753439977,6.124913899120717e-12,0.07822879852179906,0.40816326530612246,171,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-21,8254
+1.0,0.11053130189212229,0.260168451958828,0.42,0.3279459500984871,2.449965559648287e-12,0.07333812379932836,0.40816326530612246,172,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-28,7026
+1.0,0.06917021315146277,0.0,0.38,0.37411287881420296,9.799862238593149e-13,0.07465061371735272,0.39455782312925175,173,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-04-04,6412
+1.0,0.06728264676731566,0.0,0.44,0.4347510050616973,3.9199448954372595e-13,0.0732721326861316,0.39455782312925175,174,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,2021-04-11,6297
+1.0,0.10167805497311716,0.0,0.43,0.4574504815633023,1.5679779581749037e-13,0.07982724993034271,0.39455782312925175,175,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,2021-04-18,6687
+1.0,0.1734619149834527,0.0,0.48,0.48912312446006045,6.271911832699615e-14,0.06941165256412136,0.39455782312925175,176,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-04-25,8430
+1.0,0.2040432878056308,0.0,0.46,0.44466429049983563,2.5087647330798465e-14,0.06276854716898124,0.39455782312925175,177,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-02,8025
+1.0,0.20788046814877387,0.0,0.48,0.5722675873212515,1.0035058932319387e-14,0.04882242524673344,0.40136054421768713,178,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-09,8242
+1.0,0.14929264058846564,0.0,0.5,0.45913415146070335,4.014023572927755e-15,0.033618806000791895,0.40136054421768713,179,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-16,8280
+1.0,0.11694210039888364,0.0,0.51,0.39528662679579885,1.6056094291711022e-15,0.025182942030473228,0.40136054421768713,180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-23,7909
+1.0,0.055184035342337234,0.0,0.51,0.3880077087936407,6.422437716684409e-16,0.017652622780132275,0.40136054421768713,181,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-30,7574
+1.0,0.04358787034563821,0.0,0.5,0.3863265622647678,2.568975086673764e-16,0.012651962869000308,0.3673469387755103,182,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-06-06,7270
+1.0,0.03833609653008979,0.0,0.46,0.3784495643657444,1.0275900346695056e-16,0.008835476822454577,0.3673469387755103,183,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-06-13,6716
+1.0,0.06111263589867566,0.0,0.48,0.38862024435317233,4.1103601386780226e-17,0.005939200187708055,0.3673469387755103,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-06-20,6944
+1.0,0.07119833324643848,0.0,0.44,0.4039000969934476,1.644144055471209e-17,0.004967664354533589,0.3673469387755103,185,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2021-06-27,6803
+1.0,0.0659956847282599,0.0,0.45,0.4420872417106599,6.576576221884836e-18,0.004359079643941282,0.3537414965986395,186,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-04,7019
+1.0,0.12577031397293442,0.0,0.45,0.4950177419852857,2.630630488753935e-18,0.003977797656582247,0.3537414965986395,187,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-11,8254
+1.0,0.1502746019886232,0.0,0.45,0.5650602702260171,1.052252195501574e-18,0.0040621196345558795,0.3537414965986395,188,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-18,7804
+1.0,0.21001397285486328,0.0,0.42,0.594015126140436,4.209008782006296e-19,0.004952999662712088,0.3537414965986395,189,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-25,8212
+1.0,0.23464189851384848,0.0,0.46,0.5484130743981998,1.6836035128025183e-19,0.008076579020691881,0.3537414965986395,190,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-01,8378
+1.0,0.23496148203757855,0.0,0.47000000000000003,0.5324473242588711,6.734414051210074e-20,0.01220102359548914,0.3197278911564626,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-08,9496
+1.0,0.23319893582092505,0.0,0.53,0.5532778727756644,2.6937656204840295e-20,0.020152952735698258,0.3197278911564626,192,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-15,9511
+1.0,0.23262329847201318,0.0,0.49,0.7309984534528141,1.0775062481936118e-20,0.029028757460661962,0.3197278911564626,193,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-22,9569
+1.0,0.18495638415853394,0.0,0.46,0.8724050615489382,4.310024992774448e-21,0.03698435277382646,0.3197278911564626,194,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-29,7928
+1.0,0.2921700012245981,0.0,0.49,1.0,1.7240099971097793e-21,0.03982197064128697,0.3129251700680272,195,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-05,7840
+1.0,0.4172971677569805,0.0,0.48,0.8193686075762131,6.896039988439117e-22,0.03868179085216524,0.3129251700680272,196,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-12,9521
+1.0,0.5004920981884484,0.0,0.53,0.4496097944711011,2.758415995375647e-22,0.03902274493701515,0.3129251700680272,197,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-19,9451
+1.0,0.6383788968475093,0.0,0.47000000000000003,0.3701822126418114,1.1033663981502588e-22,0.03567186285580209,0.3129251700680272,198,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-26,8898
+1.0,0.6501651617929107,0.0,0.51,0.34258196039636274,4.413465592601035e-23,0.0352539191388893,0.3129251700680272,199,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-03,8441
+1.0,0.6649283374522998,0.0,0.51,0.31355701111053985,1.7653862370404143e-23,0.03635010485254652,0.28571428571428575,200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-10,8788
+1.0,0.6097114754591861,0.0,0.51,0.32306971094469733,7.061544948161657e-24,0.031323781730726925,0.28571428571428575,201,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-17,9569
+1.0,0.3964279757062242,0.0,0.51,0.33051520280988034,2.8246179792646632e-24,0.02719933715592967,0.28571428571428575,202,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-24,9008
+1.0,0.33105364706311086,0.0,0.47000000000000003,0.3259978333423606,1.1298471917058652e-24,0.025967503042923553,0.28571428571428575,203,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-31,8495
+1.0,0.31714045716637634,0.0,0.55,0.3045528431182349,4.519388766823461e-25,0.02263128565353199,0.2653061224489796,204,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-07,8807
+1.0,0.28268319082761023,0.0,0.49,0.31370309424641213,1.8077555067293845e-25,0.01786159463858867,0.2653061224489796,205,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-14,8385
+1.0,0.15774740707436136,0.0,0.51,0.37945364695975814,7.231022026917538e-26,0.016409790148260033,0.2653061224489796,206,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-21,6964
+1.0,0.2836203500514554,0.0,0.55,0.36793503370466,2.892408810767015e-26,0.01882946429880776,0.2653061224489796,207,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-28,9340
+1.0,0.33646919882766096,0.0,0.49,0.3299836196379579,1.1569635243068062e-26,0.023555161238286576,0.272108843537415,208,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-05,8632
+1.0,0.361268166630245,0.0,0.38,0.3243428164088717,4.6278540972272255e-27,0.029421037966887126,0.272108843537415,209,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-12,9271
+1.0,0.21850759166298056,0.0,0.51,0.34100191273497404,1.8511416388908902e-27,0.029549354020325262,0.272108843537415,210,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-19,7663
+1.0,0.2156152088113536,0.0,0.43,0.3876459690915292,7.404566555563562e-28,0.04853646375621416,0.272108843537415,211,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-26,7888
+1.0,0.4122692273972545,0.0,0.42,0.44121852053456856,2.961826622225425e-28,0.07303383144403221,0.272108843537415,212,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-02,11088
+1.0,0.5580863257308297,0.0,0.42,0.33648328199770844,1.18473064889017e-28,0.2914790808171166,0.2585034013605442,213,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-09,12850
+1.0,0.5441541455767391,0.0,0.45,0.5258301345263098,4.7389225955606806e-29,0.6228644542534939,0.2585034013605442,214,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-16,12768
+1.0,0.37953926965668333,0.0,0.51,0.6191133700101356,1.8955690382242722e-29,1.0,0.2585034013605442,215,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2022-01-23,11023
+1.0,0.3422525462363791,0.0,0.5,0.6600516747429145,7.582276152897087e-30,0.8603298089190655,0.2585034013605442,216,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-30,10317
+1.0,0.3679329127754763,0.0,0.49,0.6150147631969254,3.0329104611588346e-30,0.3851571321728674,0.2448979591836735,217,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-02-06,10109
+1.0,0.3530129569359208,0.0,0.49,0.5435710104633258,1.2131641844635335e-30,0.18207314748280565,0.2448979591836735,218,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-02-13,10233
+1.0,0.3628237688509028,0.0,0.48,0.5395383650448762,4.852656737854129e-31,0.08532284319045035,0.2448979591836735,219,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-02-20,10660
+1.0,0.3535562124344392,0.0,0.49,0.3713089856353334,1.941062695141646e-31,0.04778123212740684,0.2448979591836735,220,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-02-27,9862
+1.0,0.35851767100446613,0.0,0.49,0.33021424233802193,7.764250780566529e-32,0.028365180155739026,0.2448979591836735,221,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-03-06,10393
+1.0,0.3648140365425708,0.0,0.53,0.29899648842829235,3.105700312226557e-32,0.019053100849085656,0.2448979591836735,222,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-03-13,9914
+1.0,0.417768904168966,0.0,0.46,0.30801461857263196,1.242280124890568e-32,0.014096435013418193,0.2448979591836735,223,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-03-20,11027
+1.0,0.45364666714531404,0.0,0.5,0.29874033139572204,4.9691204995617213e-33,0.013440190054406007,0.2448979591836735,224,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-03-27,10066
+1.0,0.45997433293937545,0.0,0.45,0.3080341285301519,1.9876481998241388e-33,0.014672024167412121,0.2448979591836735,225,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-04-03,8722
+1.0,0.4245480429075594,0.0,0.46,0.304189689538618,7.950592799291056e-34,0.01936472555029256,0.2448979591836735,226,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,2022-04-10,7805
+1.0,0.4463068738641009,0.0,0.54,0.307818077305473,3.1802371197109226e-34,0.027822586558343475,0.2448979591836735,227,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,2022-04-17,8519
+1.0,0.6012222981571669,0.0,0.53,0.29394180576819906,1.272094847878869e-34,0.033340176856183366,0.2448979591836735,228,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-04-24,10084
+1.0,0.6804106164543928,0.0,0.5,0.28219281269675367,5.088379391460478e-35,0.04576117082899503,0.2448979591836735,229,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-01,10291
+1.0,0.62805714350389,0.0,0.54,0.30839694661979145,2.035351756529193e-35,0.05172603422739071,0.2448979591836735,230,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-08,9743
+1.0,0.7470007501508245,0.0,0.54,0.3120111152265925,8.141407025566787e-36,0.04952999662712088,0.2448979591836735,231,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-15,10759
+1.0,0.6460736106378411,0.0,0.55,0.2905779236460707,3.25656280967673e-36,0.06457597043598129,0.2448979591836735,232,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-22,9845
+1.0,0.5732108245519132,0.0,0.52,0.38068837954927237,1.3026251233207076e-36,0.080201199571791,0.2448979591836735,233,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-29,9499
+1.0,0.5996683384067256,0.0,0.5,0.3940488499594224,5.210500487782985e-37,0.09049581323048496,0.40680272108843546,234,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-06-05,10021
+1.0,0.5630659455826548,0.0,0.54,0.4539755399873685,2.0842001896133483e-37,0.09128037424293528,0.40680272108843546,235,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-06-12,10112
+1.0,0.5482324249484887,0.0,0.45,0.48814019600803654,8.336800703454939e-38,0.08289217052103649,0.40680272108843546,236,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-06-19,10034
+1.0,0.5485743918729864,0.0,0.47000000000000003,0.475428506654356,3.3347202263835196e-38,0.06987359035649866,0.40680272108843546,237,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2022-06-26,9209
+1.0,0.5559932625646005,0.0,0.43,0.510072176038165,1.333888035554951e-38,0.06264756346145385,0.40680272108843546,238,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-03,10265
+1.0,0.6089718159266746,0.0,0.45,0.44215508529036335,5.33555159223524e-39,0.0627612148230705,0.40680272108843546,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-10,10033
+1.0,0.6101706458097598,0.0,0.48,0.41550269661979555,2.1342200869095313e-39,0.07072780865510112,0.40680272108843546,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-17,9790
+1.0,0.6111403594460636,0.0,0.44,0.437146146258812,8.536874847792479e-40,0.07964760745552932,0.40680272108843546,241,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-24,9629
+1.0,0.6451477728019566,0.0,0.44,0.4975101423754845,3.4147444392713438e-40,0.0893739643061401,0.40680272108843546,242,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-31,10134
+1.0,0.7267513590970145,0.0,0.44,0.5042632593424633,1.3658922758628901e-40,0.09389435556011791,0.40680272108843546,243,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-07,12029
+1.0,0.832744074444703,0.0,0.46,0.5840915039533217,5.463514104995084e-41,0.08482790984147467,0.40680272108843546,244,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-14,12886
+1.0,0.8546151893753493,0.0,0.49,0.6374603327364593,2.1853506435415578e-41,0.07962194424484169,0.40680272108843546,245,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-21,12027
+1.0,0.9999999999999998,0.0,0.55,0.6022458246191313,8.740852589601472e-42,0.07178366646624922,0.40680272108843546,246,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-28,11375
+1.0,0.860672618209781,0.0,0.48,0.5735957859704555,3.495791051275827e-42,0.05725095687114135,0.40680272108843546,247,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-04,10824
+1.0,0.8622728019659036,0.0,0.54,0.5790428094946118,1.39776643594557e-42,0.050739833702394745,0.40680272108843546,248,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-11,12285
+1.0,0.7774120906393625,0.0,0.55,0.7618650061054455,5.585565898134668e-43,0.0440857297883885,0.40680272108843546,249,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-18,12146
+1.0,0.6580209603679659,0.0,0.52,0.8137272725878776,2.2287265136062566e-43,0.039975949905412735,0.40680272108843546,250,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-25,10881
+1.0,0.9480011027127861,0.0,0.52,0.7867690657367606,8.859907597948911e-44,0.03648941942485079,0.40680272108843546,251,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-02,11373
+1.0,0.709096498806814,0.0,0.46,0.7292818780372798,3.4889645827034517e-44,0.04076784326377381,0.40680272108843546,252,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-09,10230
+1.0,0.5414415970743589,0.0,0.45,0.6974583695681711,1.340587376605267e-44,0.04368978310920796,0.0,253,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-16,11557
+1.0,0.6081525119323576,0.0,0.54,0.6240593695822464,4.812364941659934e-45,0.041156457597043596,0.0,254,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-23,10805
+1.0,0.5960421531458853,0.0,0.45,0.5899287906913332,1.3749614119028383e-45,0.03843982343711047,0.0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-30,9709
+1.0,0.848521629204434,0.0,0.47000000000000003,0.6201930426013046,0.0,0.040723849188309305,0.0,256,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-11-06,10098