import pandas as pd
import numpy as np
import os
import holidays
from random import random
import calendar
from datetime import date
import datetime
import yaml
from ruamel.yaml import YAML
import xlwings as xw
# from xlwings.constants import DeleteShiftDirection
import sys
import time
import csv
import glob
import shutil
import collections
from yaml.representer import Representer
yaml.SafeDumper.add_representer(collections.defaultdict, Representer.represent_dict)
from ruamel.yaml import YAML
import signal
import psutil
from simple_colors import *

##########################################################

def initialization_users():
    """
    This function aggregates the initialization processes needed in the simulation.
    """

    print(blue("\nGenerate parameters in yaml file from xlsx file:", ['bold', 'underlined']), '\n')

    generate_users_yml(base=36)

    generate_plant_yml()

    membership_matrix()

    plant_operation_matrix()

##########################################################

def generate_calendar():
    """
    This function generates the calendar.

    Inputs:
        start_date      datetime, starting date to generate the calendar (from config.yml)
        end_date        datetime, ending date to generate the calendar
        delta_t         string, time interval between 2 timesteps, indicated as datetime format ("15min" or "1H")
    Outputs:
        filename_calendar file with datetime, working_day (0=monday, 6=sunday), holiday (True/False) and Tariff time slots (1,2,3)
    """

    print(blue("\nCreate calendar:", ['bold', 'underlined']), '\n')

    # getting all needed inputs from config.yml  
    config = yaml.safe_load(open("config.yml", 'r'))
    start_date = str(config['start_date'])

    project_lifetime_yrs = config['project_lifetime_yrs']

    delta_t = config['delta_t']

    end_date = str(int(start_date[:4]) + project_lifetime_yrs) + '-01-01'

    filename_giorni_tipo = config['filename_giorni_tipo']
    giorni_tipo = pd.read_csv(filename_giorni_tipo) # ???

    print("start date: ", start_date)
    print("number of years: ", project_lifetime_yrs)
    print("delta time: ", delta_t)

    # generating the calendar
    cal = pd.DataFrame({"datetime": pd.date_range(start=start_date, end=end_date, freq=delta_t)})
    cal=cal[:-1] # removing the last element which is out of the needed range

    cal['day_week']= cal.datetime.dt.dayofweek

    cal["day"] = cal.datetime.dt.day 

    #######################################################################################################

    it_holidays = holidays.IT() # getting italian holidays
    cal['holiday'] = cal['datetime'].apply(lambda x: x.date() in it_holidays)

    #######################################################################################################

    # extracting days of the week
    Lun=cal[(cal['day_week']==0) & (cal['holiday'] == False)]
    Mar=cal[(cal['day_week']==1) & (cal['holiday'] == False)]
    Mer=cal[(cal['day_week']==2) & (cal['holiday'] == False)]
    Gio=cal[(cal['day_week']==3) & (cal['holiday'] == False)]
    Ven=cal[(cal['day_week']==4) & (cal['holiday'] == False)]
    Sab=cal[(cal['day_week']==5) & (cal['holiday'] == False)]
    Dom=cal[(cal['day_week']==6) | (cal['holiday'] == True)]

    # data ranges
    ind_L=int(Lun['day'].shape[0]/96)
    ind_M=int(Mar['day'].shape[0]/96)
    ind_Me=int(Mer['day'].shape[0]/96)
    ind_G=int(Gio['day'].shape[0]/96)
    ind_V=int(Ven['day'].shape[0]/96)
    ind_S=int(Sab['day'].shape[0]/96)
    ind_D=int(Dom['day'].shape[0]/96)

    # type of day
    arr_lun=np.array([])
    arr_mar=np.array([])
    arr_mer=np.array([])
    arr_gio=np.array([])
    arr_ven=np.array([])
    arr_sab=np.array([])
    arr_dom=np.array([])

    # establishing the tariff time slot based on type of day (working day, saturday, sunday, holiday)
    for i in range(0,ind_L):
        arr_lun=np.concatenate([arr_lun,giorni_tipo['Lav']])
    #
    for i in range(0,ind_M):
        arr_mar=np.concatenate([arr_mar,giorni_tipo['Lav']])
    #
    for i in range(0,ind_Me):
        arr_mer=np.concatenate([arr_mer,giorni_tipo['Lav']])
    #
    for i in range(0,ind_G):
        arr_gio=np.concatenate([arr_gio,giorni_tipo['Lav']])
    #
    for i in range(0,ind_V):
        arr_ven=np.concatenate([arr_ven,giorni_tipo['Lav']])
    #
    for i in range(0,ind_S):
        arr_sab=np.concatenate([arr_sab,giorni_tipo['Sab']])
    #
    for i in range(0,ind_D):
        arr_dom=np.concatenate([arr_dom,giorni_tipo['Dom']])
    
    Lun.insert(4, 'fascia', arr_lun)
    Mar.insert(4, 'fascia', arr_mar) 
    Mer.insert(4, 'fascia', arr_mer) 
    Gio.insert(4, 'fascia', arr_gio)
    Ven.insert(4, 'fascia', arr_ven)
    Sab.insert(4, 'fascia', arr_sab) 
    Dom.insert(4, 'fascia', arr_dom) 
    #
    cal = pd.concat([Lun,Mar,Mer,Gio,Ven,Sab,Dom], sort=False).sort_index()
    cal.drop(columns=["day"], inplace=True)
    
    #######################################################################################################

    # preassigning value of working_day
    cal["day_flag"] = "Working_day"
    cal.loc[cal["day_week"] == 5,"day_flag"] = "Saturday" #overwrite the saturdays
    cal.loc[cal["day_week"] == 6,"day_flag"] = "Sunday" #overwrite the sundays
    cal.loc[cal["holiday"],"day_flag"] = "Sunday" #overwrite the holidays, modelled as sundays

    filename_calendar = config['filename_calendar']
    cal.to_csv(filename_calendar,index=False)

    #######################################################################################################

    # creating a calendar for the financial analysis, monthly-based
    filename_monthly_calendar = config["filename_monthly_calendar"]
    cal["month"] = cal.datetime.dt.strftime("%Y-%m")

    monthly_calendar = pd.DataFrame(data={"month": cal["month"].unique()})
    monthly_calendar["month_number"] = np.linspace(1, len(monthly_calendar),len(monthly_calendar)).astype(int)
    monthly_calendar[["month_number","month"]].to_csv(filename_monthly_calendar,index=True)

    print("\n**** Calendar successfully exported! ****\n")

##########################################################

def get_calendar():
    """ retrieves the active calendar from the file config['filename_calendar'] as a dataframe, as it is, without updating, and adjusts the datetime format
    Output:
        cal: dataframe
    """
    config = yaml.safe_load(open("config.yml", 'r'))
    cal = pd.read_csv(config['filename_calendar'])
    cal['datetime'] = pd.to_datetime(cal['datetime'], format = "%Y-%m-%d %H:%M:%S")
    return cal

##########################################################

def get_monthly_calendar():
    """ retrieves the active monthly calendar, consisting in just a series with months formatted as YYYY-MM
    Output:
        cal: dataframe
    """
    config = yaml.safe_load(open("config.yml", 'r'))
    cal = pd.read_csv(config['filename_monthly_calendar'], index_col=0)
    return cal

##########################################################

def province_to_region():
    """
    As the ARERA load profiles are region-based, this function returns the region of the selected municipality, based on the file "comuni_italiani.csv" table.
    Input must be the name of the province in English
    """
    config = yaml.safe_load(open("config.yml", 'r'))
    italian_municipalities = pd.read_csv(config["filename_comuni_italiani"], encoding='unicode_escape')

    assert (italian_municipalities["Denominazione in italiano"] == config["provincia_it"]).any(), "Location not found in comuni_italiani.csv"

    region = italian_municipalities[italian_municipalities["Denominazione in italiano"] == config["provincia_it"]]["Denominazione Regione"].iloc[0]

    return region

def province_italian_to_english():
    """
    This function translates the province name from Italian to English, based on the file "comuni_italiani.csv" table.
    Input must be the name of the province in Italian. 
    """

    config = yaml.safe_load(open("config.yml", 'r'))
    italian_municipalities = pd.read_csv(config["filename_comuni_italiani"], encoding='unicode_escape')

    assert (italian_municipalities["Denominazione in italiano"] == config["provincia_it"]).any(), "Location not found in comuni_italiani.csv"

    province = italian_municipalities[italian_municipalities["Denominazione in italiano"] == config["provincia_it"]]["Denominazione provincia in inglese"].iloc[0]

    return province

def location_italian_to_english(location_it):
    """
    This function translates the municipality name from Italian to English, based on the file "comuni_italiani.csv" table.
    Input must be the name of the municipality in Italian. 
    """
    config = yaml.safe_load(open("config.yml", 'r'))
    italian_municipalities = pd.read_csv(config["filename_comuni_italiani"], encoding='unicode_escape')

    assert (italian_municipalities["Denominazione in italiano"] == location_it).any(), "Location not found in comuni_italiani.csv"

    location_en = italian_municipalities[italian_municipalities["Denominazione in italiano"] == location_it]["Denominazione provincia in inglese"].iloc[0]

    return location_en

##########################################################

def generate_users_yml(base=36):
    """From the filename_users_CACER_xls file, we do some checks and then generates the filename_registry_user_types_yml as dictionary, without nesting.
    We create at the same time registry_user_types.yml and registry_user.yml.
    Inputs: 
        base: the base of the user IDs. can be base=10 (then users ID will be u_009, u_010, ..., u_999) or base=36 (then users ID will be u_009, u_00A, u_00B, ... , u_ZZZ). We can have base^3 users
    """

    print(blue("\nGenerating registry_user_types.yml and registry_user.yml:\n"))

    config = yaml.safe_load(open("config.yml", 'r')) # opening file config
    filename_recap = config['filename_recap']

    app = xw.App(visible = False)
    wb = xw.Book(config['filename_users_CACER_xls']) # opening file users CACER and importing the data in a dataframe
    
    df = wb.sheets["Utenti"].range('A1').options(pd.DataFrame, 
                                                header=1,
                                                index=False, 
                                                expand='table').value
    
    wb.close() # closing file
    app.quit()

    df = df[df["flag"] == True] # dropping users not activated for the current simulation

    df["grant_pnrr"] = df["grant_pnrr"].fillna(0) # removing nan, which can generate erros in the incentives calculation

    # CHECKS on imported data
    # all user type IDs must be unique
    assert len(df.user_type.unique()) == len(df.user_type), "ERROR: some user_type share the same ID! Rectify in users_CACER.xlsx so they are all unique"
    # all user type denominations must be unique
    assert len(df.denomination.unique()) == len(df.denomination), "ERROR: some user_type share the same denomination! Rectify in users_CACER.xlsx so they are all unique"

    # checking if the names are too long for excel to handle as sheetnames (will trigger errors later on)
    names_length_flag = [len(name)>31 for name in df["user_type"]] 
    assert not True in names_length_flag, "ERROR: some user_types have names with >31 characters, which will trigger Excel errors! Please rename them with less digits"

    df.set_index("user_type", inplace=True) # setting user type ID as index
    df.drop(columns=["number_type_id"],inplace=True) # removing first column, not needed
    # replacing nan and null with 0 and setting correct format, to prevent future errors
    df.num = df.num.fillna(0).astype(int) 
    df.pv = df.pv.fillna(0).astype(int)
    df.battery = df.battery.fillna(0).astype(int)
    df.wind = df.wind.fillna(0).astype(int)
    df.dummy_user = df.dummy_user.fillna(False).astype(bool)

    # initializing variables
    users_count = 1 # initializing user count
    all_users_list = {} # initializing the list of users for file registry_user.yml
    all_users_list_CACER = {} # same as above but only with users actually participaring in the CACER    
    
    # Filling the info for the users
    for user_type in df.index: # loop on the user type (which is the user type ID)
        for number_of_user in range(int(df.num[user_type])): # loop on the number of user for the selected type
            user_count_base36 = np.base_repr(users_count, base=base, padding=3)[-3:] # assuming to have max (base)^3  users
            user_category_id = config["category_id"][df.loc[user_type, "category"]] # estracting the category_id from the list (Uppercase), to identify easily the type of connection (industriale, domestico, comune, etc.)
            
            # if consumer
            if df.loc[user_type, "type"] == "consumer": 
                user_category_id = user_category_id.lower() # by convention, we indicate the category_id with lower case in case the user is a consumer
            
            # if prosumer
            if df.loc[user_type,"type"] == "producer": 
                user_category_id == "X" # by convention, we indicate a producer (only grid-connected generator, no load present) with capital "X"
            
            user_id = "u_" + user_category_id + user_count_base36 # we generate the user ID for the selected user
            
            all_users_list[user_id] = {} # we create an empty dictionary of the new user id
            all_users_list[user_id] = df.loc[user_type, :].to_dict() # we copy into the dictionary the same parameters that we find in the file user CACER.xls
            all_users_list[user_id]["user_type"] = user_type # we add also the user type id in the parameters of the users
            
            if df.loc[user_type, "flag_cacer"] == True:
                all_users_list_CACER[user_id] = {} # we create an empty dictionary of the new user id
                all_users_list_CACER[user_id] = df.loc[user_type, :].to_dict() # we copy into the dictionary the same parameters that we find in the file user CACER.xls
                all_users_list_CACER[user_id]["user_type"] = user_type # we add also the user type id in the parameters of the users

            users_count += 1 # we count the number of users generated in way to check if this exceed the maximum number of users that can be generated in base 36
    
    assert users_count < base**3, "WARNING: the number of users exceeds the maximum achievable with 3 digits. Please increase the base representation" 
    print("Total users: ", users_count-1)

    # adding the seniority level to all the users, starting from 1 from the oldest (based on entry_month in the CACER)
    # In case of users with same entry_month, no difference will be made
    # create temporary dataframe to sort plants based on commissioning month and installed capacity
    df_temp = pd.DataFrame.from_dict(all_users_list).T.sort_values(['entry_month'], ascending=True).reset_index().rename(columns={"index":"user_id"})
    df_temp["seniority_level"] = df_temp.index + 1
    df_temp.set_index("user_id", inplace=True)

    for user in list(all_users_list.keys()):
        all_users_list[user]["seniority_level"] = int(df_temp.loc[user,"seniority_level"])

    #ora invece creiamo la lista di utenti per tipologia
    user_types_list = {}
    for user_type in df.index: # loop on the user type
        if not df.flag[user_type]: continue #se non è selezionato, lo saltiamo
        user_types_list[user_type] = {}
        user_types_list[user_type] = df.loc[user_type,:].to_dict()

    # inizializziamo il file recap.yml, cancellando i dati delle passate simulazioni
    recap = {}
    with open(filename_recap, 'w') as f:
        yaml.safe_dump(recap, f)

    # save a copy of the plants sorted by seniority level in the recap file
    seniority_level_users = list(df_temp.index)
    add_to_recap_yml("users_sorted_on_seniority", seniority_level_users)

    # identifyint type of configuration
    add_to_recap_yml("type_of_cacer", config["type_of_cacer"])

    #some statistics to check
    print(len(user_types_list.keys()), " CACER members types created")

    print("List of users in active configuration:")
    prosumers = sum([user_types_list[user]["num"] for user in user_types_list.keys() if user_types_list[user]["type"] == "prosumer"])
    add_to_recap_yml("numero_prosumers", prosumers)
    print(prosumers, " prosumers")

    producers = sum([user_types_list[user]["num"] for user in user_types_list.keys() if user_types_list[user]["type"] == "producer"])
    print(producers, " producers")
    add_to_recap_yml("numero_producers", producers)

    consumers = sum([user_types_list[user]["num"] for user in user_types_list.keys() if user_types_list[user]["type"] == "consumer"])
    print(consumers, " consumers\n")
    add_to_recap_yml("numero_consumers", consumers)

    dummy_users = [user for user in user_types_list.keys() if user_types_list[user]["dummy_user"]]
    add_to_recap_yml("dummy_users", dummy_users)

    capacity_PV_list = []
    capacity_batt_list = []
    for user in user_types_list.keys():
        if user_types_list[user]["type"] == "consumer": continue
        for i in range(user_types_list[user]["num"]):
            if not pd.isna(user_types_list[user]["pv"]): capacity_PV_list.append(int(user_types_list[user]["pv"]))
            if not pd.isna(user_types_list[user]["battery"]): capacity_batt_list.append(int(user_types_list[user]["battery"]))

    print("PV capacity installed [kW]: ", capacity_PV_list)
    add_to_recap_yml("all_PV", capacity_PV_list)

    print("Battery capacity installed [kWh]: ", capacity_batt_list)
    add_to_recap_yml("all_storage", capacity_batt_list)

    add_to_recap_yml("list_prosumers", [user for user in all_users_list.keys() if all_users_list[user]["type"] == "prosumer"])
    add_to_recap_yml("list_producers", [user for user in all_users_list.keys() if all_users_list[user]["type"] == "producer"])
    add_to_recap_yml("list_consumers", [user for user in all_users_list.keys() if all_users_list[user]["type"] == "consumer"])

    add_to_recap_yml("list_prosumers_CACER", [user for user in all_users_list_CACER.keys() if all_users_list_CACER[user]["type"] == "prosumer"])
    add_to_recap_yml("list_producers_CACER", [user for user in all_users_list_CACER.keys() if all_users_list_CACER[user]["type"] == "producer"])
    add_to_recap_yml("list_consumers_CACER", [user for user in all_users_list_CACER.keys() if all_users_list_CACER[user]["type"] == "consumer"])
    
    # list(set( )) removes the duplicates from a list
    add_to_recap_yml("list_types_prosumers", list(set([all_users_list[user]["user_type"] for user in all_users_list.keys() if all_users_list[user]["type"] == "prosumer"])))
    add_to_recap_yml("list_types_producers", list(set([all_users_list[user]["user_type"] for user in all_users_list.keys() if all_users_list[user]["type"] == "producer"])))
    add_to_recap_yml("list_types_consumers", list(set([all_users_list[user]["user_type"] for user in all_users_list.keys() if all_users_list[user]["type"] == "consumer"])))

    add_to_recap_yml("list_types_prosumers_CACER", list(set([all_users_list_CACER[user]["user_type"] for user in all_users_list_CACER.keys() if all_users_list_CACER[user]["type"] == "prosumer"])))
    add_to_recap_yml("list_types_producers_CACER", list(set([all_users_list_CACER[user]["user_type"] for user in all_users_list_CACER.keys() if all_users_list_CACER[user]["type"] == "producer"])))
    add_to_recap_yml("list_types_consumers_CACER", list(set([all_users_list_CACER[user]["user_type"] for user in all_users_list_CACER.keys() if all_users_list_CACER[user]["type"] == "consumer"])))

    add_to_recap_yml("list_user_types", list(set([all_users_list[user]["user_type"] for user in all_users_list.keys()])))
    add_to_recap_yml("all_users", list(all_users_list.keys()))

    add_to_recap_yml("list_user_types_CACER", list(set([all_users_list_CACER[user]["user_type"] for user in all_users_list_CACER.keys()])))
    add_to_recap_yml("all_users_CACER", list(all_users_list_CACER.keys()))

    add_to_recap_yml("configurations", list(set([all_users_list_CACER[user]["CP"] for user in all_users_list_CACER.keys()])))
    add_to_recap_yml("stakeholders", list(set([all_users_list_CACER[user]["stakeholder"] for user in all_users_list_CACER.keys() if not pd.isnull(all_users_list_CACER[user]["stakeholder"])] )))

    add_to_recap_yml("PV_tot", sum(capacity_PV_list))
    add_to_recap_yml("batt_tot", sum(capacity_batt_list))

    add_to_recap_yml("total_CACER_members", len(all_users_list_CACER.keys()))
    add_to_recap_yml("total_non_dummy_CACER_members", len([user for user in all_users_list_CACER.keys() if not all_users_list_CACER[user]["dummy_user"]]))
    
    if capacity_PV_list != []:
        add_to_recap_yml("PV_max", max(capacity_PV_list))
        add_to_recap_yml("PV_min", min(capacity_PV_list))
    else:
        add_to_recap_yml("PV_max", 0)
        add_to_recap_yml("PV_min", 0)

    if capacity_batt_list != []:
        add_to_recap_yml("batt_max", max(capacity_batt_list))
        add_to_recap_yml("batt_min", min(capacity_batt_list))
    else: 
        add_to_recap_yml("batt_max", 0)
        add_to_recap_yml("batt_min", 0)

    add_to_recap_yml("total_CACER_users", len(all_users_list_CACER.keys()))
    add_to_recap_yml("total_grid_users", len(all_users_list.keys()))

    #saving yml
    with open(config['filename_registry_user_types_yml'], "w") as f:
        yaml.safe_dump(user_types_list, f)

    with open(config["filename_registry_users_yml"], "w") as f:
        yaml.safe_dump(all_users_list, f)
    
    #saving as csv
    pd.DataFrame.from_dict(all_users_list).T.to_csv(config["filename_registry_users_csv"])

    print("\n**** Registry users completed! ****")

##########################################################

def generate_plant_yml():
    """from the excel file, generates the yml as dictionary, without nesting"""

    print(blue("\nGenerating plant yml:"))

    config = yaml.safe_load(open("config.yml", 'r'))
    recap = yaml.safe_load(open(config['filename_recap'], 'r'))
    users = yaml.safe_load(open(config['filename_registry_users_yml'], 'r'))

    plant_registry = {}
    for user in recap["list_prosumers"] + recap["list_producers"]: # loop over users with generation plant
        plant = user.replace("u_","p_") # assigning the plant name, replacing "u" of "user" with "p" of "plant"

        plant_registry[plant] = users[user]

        plant_registry[plant]["titolare_POD"] = user
        plant_registry[plant]["installed_capacity"] = plant_registry[plant]["pv"] + plant_registry[plant]["wind"]

        if pd.isna(plant_registry[plant]["grant_pnrr"]):
            plant_registry[plant]["grant_pnrr"] = 0

        if pd.isna(plant_registry[plant]["grant_private"]):
            plant_registry[plant]["grant_private"] = 0
        
        if pd.isna(plant_registry[plant]["debt"]):
            plant_registry[plant]["debt"] = 0

        # introducing a stop on the disbursement month, as if not correctly given generates errors in the plant_operation_matrix
        assert not (np.isnan(plant_registry[plant]["disbursement_month"]) or plant_registry[plant]["disbursement_month"] == 0), f"ERROR: disbursement_month given for {plant_registry[plant]['user_type']} is NAN or 0. Provide a value >0 in the users CACER file"

    # adding the seniority level to all the plants, starting from 1 from the oldest (based on commissioning date), needed by the GSE to calculate the energy generated by which plant and thus calculate the incentive.
    # In case of plants with same commissioning date, the one with lower installed capacity will have the priority (so to simulate the maximize the incentives)
    # create temporary dataframe to sort plants based on commissioning month and installed capacity
    df_plant = pd.DataFrame.from_dict(plant_registry).T.sort_values(['commissioning_month', 'installed_capacity'], ascending=[True, True]).reset_index().rename(columns={"index":"plant_id"})
    df_plant["seniority_level"] = df_plant.index + 1
    df_plant.set_index("plant_id", inplace=True)

    for plant in list(plant_registry.keys()):
        plant_registry[plant]["seniority_level"] = int(df_plant.loc[plant,"seniority_level"])

    # save a copy of the plants sorted by seniority level in the recap file
    seniority_level_plants = list(df_plant.index)
    add_to_recap_yml("plants_sorted_by_seniority", seniority_level_plants)

    #saving as yml
    with open(config["filename_registry_plants_yml"], "w") as f:
        yaml.safe_dump(plant_registry, f)
    
    #saving as csv
    df_plant.to_csv(config["filename_registry_plants_csv"])

    print("\n**** Registry plants completed! ****")

##########################################################

def check_file_status(filename):
    """Before running time-consuming functions, juct check that the output file is not being used by other applications, to avoid Permission Denied error at the end of the run"""
    try: open(filename, "w")
    except: 
        print("ERROR: " + filename + " file is open! Close it and rerun!")
        sys.exit()

##########################################################

def upsample_arera_dataframe_to_15min(load_user):
    """ function to upsample from 1hr to 15min the ARERA daily load profiles (only 24 rows, one per hour!). 
    The dataframe can have multiple columns (working day, sunday, saturday) but only 24 hourly rows, to avoid mixing different profiles
    Inputs:
        load_user: dataframe arera profile to upsample
    """
    #the load_user["Hour"] updating below creates a SettingWithCopyWarning, which seems to be a false positive. We can deactivate it temporarily to avoid getting it in the output
    if pd.options.mode.chained_assignment == "warn": 
        pd.options.mode.chained_assignment = None  # default='warn', disabled= None
    
    load_user["Hour"] = pd.to_datetime(load_user["Hour"], format="%H:%M:%S")
    load_user_pivot = pd.pivot_table(load_user, values='Prelievo medio Orario Regionale (kWh)', index=['Hour'],
                        columns=['Working day'])

    # creating a new row which was needed as resampling boundary at the end of the dataframe
    new_fake_datapoint = pd.DataFrame(load_user_pivot[-1:].values, index=load_user_pivot.index[-1:] + datetime.timedelta(hours=1), columns=load_user_pivot.columns)
    # load_user_pivot = load_user_pivot.append(new_fake_datapoint)
    load_user_pivot = pd.concat((load_user_pivot, new_fake_datapoint), axis = 0)
    resampled_df = load_user_pivot.resample('15min', offset="1H", closed='left').mean().ffill() / 4 #resampling from 1hr to 15 min, and dividing the energy by 4
    resampled_df = resampled_df[:-1] # removing the last row which was needed as resampling boundary
    resampled_df.reset_index(inplace=True)
    resampled_df.Hour = resampled_df.Hour.dt.time
    pd.options.mode.chained_assignment = "warn"  # default='warn', disabled= None
    return resampled_df

##########################################################

def load_profile_single_user(df, user):
    """gnerating the load profile for a single user.
    Inputs:
        df                  calendar dataframe with all the needed columns, and previous load profiles from other users
        user                user_id, same user id that we have in registry_users.keys()
    Outputs: 
        output_all_users    list with user profiles, updated with new one just computed
    """

    config = yaml.safe_load(open("config.yml", 'r'))
    user_types_set = yaml.safe_load(open(config["filename_registry_user_types_yml"], 'r')) # file yaml con i parametri delle varie categorie di utenza
    load_profile_id = user_types_set[user]["load_profile_id"] # extract load profile id from registry_user_types.yml
    power_range = user_types_set[user]["power_range"] # extract power range from registry_user_types.yml

    # CASE 1 - arera load profile
    if load_profile_id == "arera":
        for month in range(1,13): # month as integer from 1 to 12
            
            # we extract arera load for selected region, selected power range and selected month (we need to check also that this load profile exist!!!)
            load_user_month = user_load_arera[(user_load_arera["Regione"] == region) & (user_load_arera["Classe potenza"] == power_range) & (user_load_arera["Mese"] == month)]
            if delta_t == "15Min": 
                load_user_month = upsample_arera_dataframe_to_15min(load_user_month) # arera is in 1hr 
                load_user_month['month'] = month # set month in month column
                load_user_month.index.names = ['Quarter'] # set name index as 'Quarter'

            if month != 1:
                load_user = pd.concat([load_user, load_user_month], ignore_index=True) # concat all month in a single column
            else: 
                load_user = load_user_month # initialization (only for the first month)
        
        load_user.set_index(["Hour", "month"], inplace=True) # set index as a combination of "Hour" and "month" columns

    # CASE 2 - real load profile
    elif load_profile_id == "real profile":
        load_user = real_profile_df[user] # real profile for Riccomassimo CER
        load_user = pd.DataFrame(load_user)

    # CASE 3 - emulated load profile
    elif load_profile_id == "emulated profile":
        load_user = emulated_load_profile_df[user] # emulated load profile for Riccomassimo CER
        load_user = pd.DataFrame(load_user)

    # CASE 4 - other load profiles
    else: # when we have our profiles given as input
        load_user = wb.sheets[load_profile_id].range('A1').options(pd.DataFrame, header=1, index=True, expand='table').value
        load_user["Hour"] = pd.date_range("00:00", "23:45", freq = "15min").time # ATTENZIONE, avevo problemi a importare correttamente il formato del time della colonna HOUR, quindi per semplicita ne ho creato uno nuovo, assumendo che tanto va da 00:00 a 23:45
        load_user.set_index("Hour", inplace = True)

    # If not real profile and not emulated profile and rand_factor is different by 0, we just consider updating_random_variation_flag = True! 
    # So the random variation is considered in arera profile cases and other profiles cases!
    updating_random_variation_flag = rand_factor != 0 and load_profile_id != "real profile" and load_profile_id != "emulated profile"

    # If not arera profile and not real profile and not emulated profile, we just define a week_factor variation imported from external file! 
    if load_profile_id != "arera" and load_profile_id != "real profile" and load_profile_id != "emulated profile":
        week_factor = wb.sheets[load_profile_id].range('P1').options(pd.DataFrame, header=1, index=True, expand='table').value#.to_dict()

    # random factor for peak fluctuation: update the random_variation for each datapoint, as fluctuation + or - x% if (updating_random_variation_flag == True). If not, random_variation will remain 0 
    df["random_variation"] = 0 # initialize the random variation to zero value
    if updating_random_variation_flag:
        df["random"] = np.random.uniform(0, 1, [len(df),1]) # random values from 0 to 1
        df["random_variation"] = rand_factor / 100 * (df["random"] * 2 - 1) # random values between -rand_factor to +rand_factor


    if load_profile_id == "arera":
        # we prepare the load profile for the merge, stacking the hour and the day_flag, and calling the values "load_active"
        load_user_stack = load_user.stack().reset_index().rename(columns={"Working day":'day_flag', 
                                                        "Hour":"hour", 
                                                        0:"load_active"})
        df = pd.merge(df, load_user_stack, on=["hour", "month", 'day_flag'], how='inner').sort_values("datetime").reset_index(drop = True)

    elif load_profile_id == "real profile" or load_profile_id == "emulated profile":
        
        load_user = load_user.reset_index()
        load_user = load_user.drop(columns=['datetime'])
        load_user = load_user.rename(columns={user:"load_active"})
        df = pd.concat([df, load_user], axis=1)
        df = df.sort_values("datetime").reset_index(drop = True)

    else:
        
        # Arera case has the monthly profiles, while the other cases have the hourly profiles non depending on the month. Thus the method is different, as the structuree
        # of theload_user in this case has also the "month" column.
        # we prepare the load profile for the merge, stacking the hour and the day_flag, and calling the values "load_active"
        load_user_stack = load_user.stack().reset_index().rename(columns={"level_1":'day_flag', 
                                                        "Hour":"hour", 
                                                        0:"load_active"})
        
        # updating the df with the load_active values corresponding to the hour and day_flag
        df = pd.merge(df, load_user_stack, on=["hour", 'day_flag'], how = 'inner').sort_values("datetime").reset_index(drop = True)

    # if not arera profile and not real profile and not emulated profile, we update week_factor_active and year_factor_active!
    if load_profile_id != "arera" and load_profile_id != "real profile" and load_profile_id != "emulated profile": 

        week_factor_stack = week_factor.stack().reset_index().rename(columns={"level_1":'day_flag', 
                                                    "Week":"week",
                                                    0:"weekly_correction"})
        
        # updating the df with the load_active values corresponding to the week and day_flag
        df = pd.merge(df, week_factor_stack, on=["week", 'day_flag'], how='inner').sort_values("datetime").reset_index(drop=True)
        
        # updating the df with the load_active values corresponding to the year
        year_factor_stack = year_factor["yearly_correction"]
        df = pd.merge(df, year_factor_stack, on=["year"], how='inner').sort_values("datetime").reset_index(drop=True)

    else:
        df["weekly_correction"] = 1
        df["yearly_correction"] = 1

    # CALCULATION
    # we calculate the energy consumption for the specific datapoint
    df["load_active_adjusted"] = df["load_active"] * df["weekly_correction"] * df["yearly_correction"] * (1 + df["random_variation"])

    print("user created: ", blue(user), "\t\t user type: ", load_profile_id )

    return df.set_index("datetime")["load_active_adjusted"] #returning the load adjusted with the datetime as the unique index, same format of the df_results

##########################################################

def load_profile_all_users():
    
    """
    Loop over all user types, to create the 15min or 1H load profiles.
    Exports:

    """

    print(blue("\nGenerate load profile for user types added:", ['bold', 'underlined']), '\n')

    global xls, month_factor, year_factor, user_load_arera, region, delta_t, wb, output_all_users, rand_factor, real_profile_df, emulated_load_profile_df

    ########### INPUTS ##############

    config = yaml.safe_load(open("config.yml", 'r'))
    filename_user_load_arera = config['filename_user_load_arera']
    filename_carico_input = config["filename_carico_input"]
    filename_registry_user_types_yml = config["filename_registry_user_types_yml"]

    check_file_status(config["filename_carichi"]) # check if "carichi.xlsx" file is being used by other apps. If not, it'd crash at the very end of the run
    region = province_to_region() # region of Italy
    rand_factor = config['rand_factor'] # random factor for peak fluctuation
    print("Random factor: " + str(rand_factor) + " %")
    delta_t = config['delta_t'] 

    user_types_set = yaml.safe_load(open(filename_registry_user_types_yml, 'r')) # file yaml with user type data
    users_consuming_energy = [user for user in user_types_set.keys() if user_types_set[user]["consuming"]] # we extract the list of user type of which consuming is true, excluding the prosumers which don't have load profile
    load_profiles_list = [user_types_set[user]["load_profile_id"] for user in users_consuming_energy] # we extract the load profile list of user type to simulate (the values can be arera, real profile, emulated profile, etc.)

    print(len(users_consuming_energy), "user types consuming found")

    df = get_calendar() # importing the calendar
    df.drop(columns=["fascia"], inplace=True)

    # generating columns needed for calculation  
    df["hour"] = df["datetime"].dt.time # create hour column
    df["week"] = df["datetime"].dt.isocalendar().week # create week column
    df["month"] = df["datetime"].dt.month # create month column
    df["year"] = df["datetime"].dt.year # create year column

    # import load profile arera
    if "arera" in load_profiles_list:
        user_load_arera = pd.read_csv(filename_user_load_arera) # si importano i profili di consumo arera

        user_load_arera["Hour"] = pd.to_datetime(user_load_arera.Ora, format="%H").dt.time # create the Hour column in the arera df with the correct format
        
        # replace day type with the correct format
        vals_to_replace = {"Giorno feriale": "Working_day", "Sabato": "Saturday", "Domenica": "Sunday"} 
        user_load_arera["Working day"] = user_load_arera["Working day"].map(vals_to_replace) 

    # import real load profile
    if "real profile" in load_profiles_list:
        filename = config["filename_load"]
        real_profile_df = pd.read_csv(filename, index_col = 'datetime')

        first_year = config['start_date'].year
        num_years = config['project_lifetime_yrs']

        if num_years == 1:
            real_profile_df.index = pd.to_datetime(real_profile_df.index)
            real_profile_df = real_profile_df[real_profile_df.index.year == first_year]

    # import emulated load profile
    if "emulated profile" in load_profiles_list:
        filename = config["filename_emulated_load_profile"]
        emulated_load_profile_df = pd.read_csv(filename, index_col = 'datetime')

    # import other load profiles
    app = xw.App(visible = False)
    wb = xw.Book(filename_carico_input)

    year_factor = wb.sheets['Yearly_Variation'].range('A1').options(pd.DataFrame, header=1, index=True, expand='table').value

    # check on load profile name, to make sure they are all given
    for user in users_consuming_energy:
        load_profile_id = user_types_set[user]["load_profile_id"]
        if load_profile_id in ["arera", "real profile", "emulated profile", None]: 
            continue
        else:
            # if profile does not come from any of the above sources, it must come from the input file, meaning the load_profile_id must be in one of the document's sheets
            sheets = [sheet.name for sheet in wb.sheets] 
            # If not present, then error must be triggered
            assert load_profile_id in sheets, print("ATTENTION: the", load_profile_id, "load profile given as input was not found!")

    #Start the loop over the consuming user types

    # initializing the results datafame, as copy of df (calendar imported) but with "datetime" as index
    df_results = df.copy(deep=True)
    df_results = df_results.set_index("datetime")

    for user in users_consuming_energy: # loop on user types

        # we generate the load profile of the selected user type for all the timestep over the project lifetime and append it in the output_all_users
        df_results[user] = load_profile_single_user(df, user)
        assert not df_results[user].isna().any(), "ERROR: Indexing failure! NaN found in " + user

    wb.close()
    app.quit()

    # #################### EXIT ################
    df_results.drop(columns=["hour", "week", "month", "year", "holiday", "day_week", "day_flag"]).to_csv(config["filename_carichi"])

    print("\n**** Load profiles successfully exported! ****")

###############################################################################################################################

def check_folder_exists(folder_path):
    """check if a folder exists. If not, it creates it. It is needed before running the time-consuming functions to verify the output folder actually exists"""
    if not os.path.isdir(folder_path):
        os.makedirs(folder_path)

###############################################################################################################################

def clear_folder_content(folder_directory):
    """removing all content in a folder"""
    files = glob.glob(folder_directory+'*')
    for f in files:
        os.remove(f) #deleting all files
    print("All files in " + folder_directory + " folder deleted")

###############################################################################################################################

def add_to_recap_yml(key, value):
    """saving some value under a given key in the recap.yml dictionary. Needed for reporting and recap purposes"""
    config = yaml.safe_load(open("config.yml", 'r'))
    filename_recap = config['filename_recap']
    
    recap = yaml.safe_load(open(filename_recap, 'r'))
    
    recap[key] = value

    with open(filename_recap, 'w') as f:
        yaml.safe_dump(recap, f)

def update_irr_on_recap_yml(user,irr_value):
    """saving some irr_value under a given user in the recap.yml dictionary. Needed for reporting and recap purposes"""
    config = yaml.safe_load(open("config.yml", 'r'))
    recap = yaml.safe_load(open(config['filename_recap'], 'r'))

    if "irr" not in recap.keys(): recap["irr"] = None
    
    recap["irr"][user] = irr_value
    
    with open(config['filename_recap'], 'w') as f:
        yaml.safe_dump(recap, f)

################################################################################################################################
# UNNEEDED
# def add_to_config_yml(key,value):
#     #dati utili per report
#     config = yaml.safe_load(open("config.yml", 'r'))
    
#     # modifichiamo quello che ci serve
#     config[key] = value

#     # salviamo
#     with open("config.yml", 'w') as f:
#         yaml.safe_dump(config, f)

###############################################################################################################################
# UNNEEDED
# def add_to_file_yml(path, filename, key, value):
       
#     #importiamo
#     diz = yaml.safe_load(open(path + filename, 'r'))
    
#     # modifichiamo quello che ci serve
#     diz[key] = value

#     # salviamo
#     with open(path + filename, 'w') as f:
#         yaml.safe_dump(diz, f)

###############################################################################################################################

def add_to_file_yml(path, filename, key, value):
    yaml = YAML()
    yaml.preserve_quotes = True

    # Read the YAML file
    with open(path, 'r', encoding='utf-8') as file:
        data = yaml.load(file)

    # Ensure the structure exists
    if filename:
        if filename not in data:
            data[filename] = {}
        data[filename][key] = value
    else:
        data[key] = value

    # Write back to the file without losing comments
    with open(path, 'w', encoding='utf-8') as file:
        yaml.dump(data, file)

##########################################################

def edit_file_yml_preserving_comments(file_path, key, value): 
    """
    Edit a YAML file, preserving comments.
    Inputs:
        file_path: path to the YAML file to edit
        key: key to be modified
        value: new value for the key
    """
    yaml = YAML()
    
    # Read the file and preserve comments
    with open(file_path, 'r') as file:
        data = yaml.load(file)
    
    # Modify the data as needed
    data[key] = value
    
    # Write back to the file, keeping comments
    with open(file_path, 'w') as file:
        yaml.dump(data, file)

################################################################################################################################

def edit_users_CACER(variable, user_type, value):
    """editing the users_CACER excel file, directly on the file
    WARNING: this function, if used in large for loops, can become very slow. Sometimes leaves some background activity that slows down the laptop (check Task Manager) and a restart could be recommended.
    """

    config = yaml.safe_load(open("config.yml", 'r'))
    app = xw.App(visible = False)
    wb = xw.Book(config["filename_users_CACER_xls"])
    num_rows = len(wb.sheets["Utenti"]["A1"].options(pd.Series, header=1, index=True, expand='table').value)

    col = 1
    while col < 100: # modificare se dovessimo avere piu di 100 colonne
        if wb.sheets["Utenti"].range((1,col)).value == variable: break
        else: col += 1

    # looking for user_type column (position might change over time)
    col_user_type = 1
    while col_user_type < 100: # edit in case we exceed the 100 columns
        if wb.sheets["Utenti"].range((1,col_user_type)).value == "user_type": break
        else: col_user_type += 1

    row = 1
    while row < num_rows+2: 
        if wb.sheets["Utenti"].range((row,col_user_type)).value == user_type: break
        else: row += 1

    # print("The Row is: "+str(row)+" and the column is "+str(col))
    wb.sheets["Utenti"].range((row,col)).value = value

    wb.save()
    wb.close()
    app.quit()

################################################################################################################################
def clear_users_utenti_CACER():
    """resets the num column in the filename_users_CACER_xls to a series of nan. Could be needed f.i. when performing a sensitivity analysis on CACER members numerosity, or when activating or removing some specific users"""

    config = yaml.safe_load(open("config.yml", 'r'))
    app = xw.App(visible = False)
    wb = xw.Book(config["filename_users_CACER_xls"])
    num_rows = len(wb.sheets["Utenti"]["A1"].options(pd.Series, header=1, index=True, expand='table').value)

    # looking for num column (position might change over time)
    col_num = 1
    while col_num < 100: # edit in case we exceed the 100 columns
        if wb.sheets["Utenti"].range((1,col_num)).value == "num": break
        else: col_num += 1

    row = 2
    while row < num_rows+2: # modificare se dovessimo avere piu di 100 righe
        wb.sheets["Utenti"].range((row,col_num)).clear_contents()
        row += 1

    wb.save()
    wb.close()
    app.quit()

def edit_incentive_repartition_scheme(value):
    """edits the incentive repartition scheme in the "Scenario" sheet in the filename_input_FM_excel. It is sometimes needed when comparing different repartition schemes"""

    config = yaml.safe_load(open("config.yml", 'r'))
    
    ###### WARNING: the use of xw.App can cause issues if the file is already opened... To be fixed ########################
    app = xw.App(visible = False)
    wb = xw.Book(config["filename_input_FM_excel"])
    wb.sheets["Scenario"].range("incentives_repartition_scheme").value = value

    wb.save()
    wb.close()
    app.quit()

def edit_opex_repartition_scheme(value):
    """edits the OPEX repartition scheme in the "Scenario" sheet in the filename_input_FM_excel. It is sometimes needed when comparing different repartition schemes"""

    config = yaml.safe_load(open("config.yml", 'r'))
    
    ###### WARNING: the use of xw.App can cause issues if the file is already opened... To be fixed ########################
    # app = xw.App(visible = False)
    wb = xw.Book(config["filename_input_FM_excel"])
    wb.sheets["Scenario"].range("opex_repartition_scheme").value = value

    wb.save()
    # wb.close()
    # app.quit()

def edit_surplus_repartition_scheme(value):
    """edits the SURPLUS repartition scheme in the "Scenario" sheet in the filename_input_FM_excel. It is sometimes needed when comparing different repartition schemes"""

    config = yaml.safe_load(open("config.yml", 'r'))
    
    ###### WARNING: the use of xw.App can cause issues if the file is already opened... To be fixed ########################
    app = xw.App(visible = False)
    wb = xw.Book(config["filename_input_FM_excel"])
    wb.sheets["Scenario"].range("surplus_repartition_scheme").value = value

    wb.save()
    wb.close()
    app.quit()

# UNNEEDED
# def duplicate_and_rename_folder(source_folder_path, new_folder_path):
#     # Get parent and child directories
#     parent_dir = os.path.dirname(source_folder_path)

#     # Create the new directory
#     # new_folder_path = os.path.join(parent_dir, child_dir)
#     os.makedirs(new_folder_path, exist_ok=True)

#     # Copy all the files and directories from the old folder to the new one
#     for root, dirs, files in os.walk(source_folder_path):
#         for file in files:
#             src_file_path = os.path.join(root, file)
#             dst_file_path = os.path.join(new_folder_path, file)
#             shutil.copy2(src_file_path, dst_file_path)
#     print(source_folder_path + " folder successfully duplicated as " + new_folder_path)



################################################################################################################################

# DA SISTEMARE!!!
# # UNNEEDED
# def aggiungi_nuovi_utenti_yml(base=36):
#     """from the excel file, generates the yml as dictionary, without nesting.
#     We create at the same time registry_user_types.yml and registry_user.yml.
#     """

#     config = yaml.safe_load(open("config.yml", 'r'))
#     filename_users_CACER_xls = config['filename_users_CACER_xls'] # si importa il nome del file .xls contenente gli utenti da simulare (tutte le tipologie di utenza!)
#     filename_registry_user_types_yml = config['filename_registry_user_types_yml'] # si importa il file yaml in cui si salveranno le caratteristiche delle varie tipologie di utenza
#     filename_recap = config['filename_recap'] # si importa il file recap

#     #########################################################################

#     # si apre il file excel e si importa il foglio "utenti" in un df
#     # app = xw.App(visible = False)
#     wb = xw.Book(filename_users_CACER_xls)
#     df = wb.sheets["Utenti"].range('A1').options(pd.DataFrame, 
#                                                 header=1,
#                                                 index=False, 
#                                                 expand='table').value
    
#     wb.close() # si chiude il file excel
#     # app.quit()

#     #########################################################################

#     assert len(df.user_type.unique()) == len(df.user_type), "some user_type share the same ID!" # si verifica che lo user ID sia unico per ogni utente
    
#     #########################################################################

#     df.set_index("user_type", inplace=True) # si setta lo user ID come index
#     df.drop(columns=["number_type_id"],inplace=True) # si elimina la prima colonna
#     df.num = df.num.fillna(0).astype(int) # si annulla la numerosità di tutti gli utenti

#     #########################################################################
    
#     # creiamo la lista di utenti della configurazione in esame che useremo per creare il file registry_user_yml
#     users_count = 0 # salviamo il numero di utenti
#     all_users_list = {} # creiamo la lista di utenti per nome che servirà per creare il file refistry_user.yml
    
#     for user_type in df.index: # loop on the user type (ID)
#         for number_of_user in range(int(df.num[user_type])): # loop on the number of user for the selected type
#             user_count_base36 = np.base_repr(users_count, base=base, padding=3)[-3:] # assuming to have max (36)^3 =  users
#             user_category_id = config["category_id"][df.loc[user_type, "category"]] # estracting the category_id from the list (Uppercase), to identify easily the type of connection (industriale, domestico, comune, etc.)
            
#             # if consumer
#             if df.loc[user_type, "type"] == "consumer": 
#                 user_category_id = user_category_id.lower() # by convention, we indicate the category_id with lower case in case the user is a consumer
            
#             # if prosumer
#             if df.loc[user_type,"type"] == "producer": 
#                 user_category_id == "X" # by convention, we indicate a producer (only grid-connected generator, no load present) with capital "X"
            
#             user_id = "u_" + user_category_id + user_count_base36 # we generate the user id for the selected user
            
#             all_users_list[user_id] = {} # we create an empty dictionary of the new user id
#             all_users_list[user_id] = df.loc[user_type, :].to_dict() # we copy into the dictionary the same parameters that we find in the file user CACER.xls
#             all_users_list[user_id]["user_type"] = user_type # we add also the user type id in the parameters of the users
            
#             users_count += 1 # we count the number of users generated in way to check if this exceed the maximum number of users that can be generated in base 36
    
#     assert users_count < base**3, "WARNING: the number of users exceeds the maximum achievable with 3 digits. Please increase the base representation" 
#     print("Total users: ", users_count)

#     #########################################################################

#     # creiamo la lista di user_types_list per tipologia che useremo per creare il file registry_user_types.yml
#     user_types_list = {}
#     for user_type in df.index: # loop on the user type
#         if not df.flag[user_type]: continue #se non è selezionato, lo saltiamo
#         user_types_list[user_type] = {}
#         user_types_list[user_type] = df.loc[user_type,:].to_dict()

#     #########################################################################

#     # saving yml with data of user types
#     with open(filename_registry_user_types_yml, "w") as f:
#         yaml.safe_dump(user_types_list, f)

#     #saving yml with data of configuration users
#     with open(config["filename_utenze_users_yml"], "w") as f:
#         yaml.safe_dump(all_users_list, f)

#     #########################################################################

#     # si popola il file recap con varie statistiche

#     recap = {} # inizializziamo il file recap.yml, cancellando i dati delle passate simulazioni

#     # ripuliamo il file recap al fine di ripopolarlo nuovamente
#     with open(filename_recap, 'w') as f:
#         yaml.safe_dump(recap, f)

#     print(len(user_types_list.keys()), " CER members types created")

#     print("List of users in active configuration:")
#     prosumers = sum([user_types_list[user]["num"] for user in user_types_list.keys() if user_types_list[user]["type"] == "prosumer"])
#     add_to_recap_yml("numero_prosumers",prosumers)
#     print(prosumers, " prosumers")

#     producers = sum([user_types_list[user]["num"] for user in user_types_list.keys() if user_types_list[user]["type"] == "producer"])
#     print(producers, " producers")
#     add_to_recap_yml("numero_producers",producers)

#     consumers = sum([user_types_list[user]["num"] for user in user_types_list.keys() if user_types_list[user]["type"] == "consumer"])
#     print(consumers, " consumers")
#     add_to_recap_yml("numero_consumers",consumers)

#     #########################################################################

#     capacity_PV_list = []
#     capacity_batt_list = []
#     for user in user_types_list.keys():
#         if user_types_list[user]["type"] == "consumer": continue
#         for i in range(user_types_list[user]["num"]):
#             if not pd.isna(user_types_list[user]["pv"]): capacity_PV_list.append(int(user_types_list[user]["pv"]))
#             if not pd.isna(user_types_list[user]["battery"]): capacity_batt_list.append(int(user_types_list[user]["battery"]))

#     print("PV capacity installed [kW]: ", capacity_PV_list)
#     add_to_recap_yml("all_PV", capacity_PV_list)

#     print("Battery capacity installed [kWh]: ", capacity_batt_list)
#     add_to_recap_yml("all_storage", capacity_batt_list)

#     #########################################################################

#     add_to_recap_yml("list_prosumers",[user for user in all_users_list.keys() if all_users_list[user]["type"] == "prosumer"])
#     add_to_recap_yml("list_producers",[user for user in all_users_list.keys() if all_users_list[user]["type"] == "producer"])
#     add_to_recap_yml("list_consumers",[user for user in all_users_list.keys() if all_users_list[user]["type"] == "consumer"])

#     #########################################################################
    
#     # list(set( )) serve per rimuovere i doppioni dalla lista
#     add_to_recap_yml("list_types_prosumers",list(set([all_users_list[user]["user_type"] for user in all_users_list.keys() if all_users_list[user]["type"] == "prosumer"])))
#     add_to_recap_yml("list_types_producers",list(set([all_users_list[user]["user_type"] for user in all_users_list.keys() if all_users_list[user]["type"] == "producer"])))
#     add_to_recap_yml("list_types_consumers",list(set([all_users_list[user]["user_type"] for user in all_users_list.keys() if all_users_list[user]["type"] == "consumer"])))

#     add_to_recap_yml("list_user_types", list(set([all_users_list[user]["user_type"] for user in all_users_list.keys()])))
#     add_to_recap_yml("all_users", list(all_users_list.keys()))

#     add_to_recap_yml("PV_tot", sum(capacity_PV_list))
#     add_to_recap_yml("batt_tot", sum(capacity_batt_list))
#     add_to_recap_yml("PV_max", max(capacity_PV_list))
#     add_to_recap_yml("PV_min", min(capacity_PV_list))

#     if capacity_batt_list != []:
#         add_to_recap_yml("batt_max", max(capacity_batt_list))
#         add_to_recap_yml("batt_min", min(capacity_batt_list))
#     else: 
#         add_to_recap_yml("batt_max", 0)
#         add_to_recap_yml("batt_min", 0)
        
#     add_to_recap_yml("total_CACER_members", len(all_users_list.keys()))

#     print("Export complete")

################################################################################################################################



def plant_operation_matrix():
    """Generating the plant activity matrix, which reports the activity or inactivity for each plant in the CACER in each month time for the project lifetime, as 1 or 0.
    It is needed to check whether the plant is operational and generating energy for the community, incentives and opex.
    If the plant exits the CACER, then it will be considered inactive for the purpose of generating value for the CACER, thus 0 from the exit month.
    IMPORTANT: plant being operational means it produces power, not necessarily it generates shared energy and thus incentives (can be active even after expiring of incentivation contract of 20 yrs). 
    Each cash flow will be evaluated separately in each dedicated function
    df has plants on index and month_number on columns"""

    print(blue("\nGenerating plant operation matrix:"))

    config = yaml.safe_load(open("config.yml", 'r'))
    user_type_set = yaml.safe_load(open(config["filename_registry_user_types_yml"], 'r'))
    user_types_producing = [user_type for user_type in user_type_set if user_type_set[user_type]["producing"]]

    plants_set = yaml.safe_load(open(config["filename_registry_plants_yml"], 'r'))
    
    writer = pd.ExcelWriter(config["filename_plant_operation_matrix"], engine = 'xlsxwriter')

    df = get_monthly_calendar().set_index("month_number")

    for user_type in user_types_producing:

        commissioning_month = user_type_set[user_type]["commissioning_month"]
        exit_month = user_type_set[user_type]["exit_month"]

        if exit_month == "end":
            exit_month = df.index[-1] + 1

        df[user_type] = [1*((month >= commissioning_month) and (month < exit_month)) for month in df.index ] # for each month, 1 meaning plant is operative (thus opex is applicable); 0 means not operative

    df.T.to_excel(writer, sheet_name= "plant_type_operation_matrix") #saving for the record

    df = get_monthly_calendar().set_index("month_number")

    for plant in plants_set:

        commissioning_month = plants_set[plant]["commissioning_month"]
        exit_month = plants_set[plant]["exit_month"]

        if exit_month == "end":
            exit_month = df.index[-1] + 1

        df[plant] = [1*((month >= commissioning_month) and (month < exit_month)) for month in df.index ] # for each month, 1 meaning plant is operative (thus opex is applicable); 0 means not operative

    df.T.to_excel(writer, sheet_name= "plant_operation_matrix") #saving for the record

    writer.close()

    print("\n**** Plant Operation Matrix created! ****")

################################################################################################################################

def membership_matrix():
    """generating the membership matrix, which reports the precence or absense for each user in the CACER in each month time for the project lifetime, as 1 or 0.
    It is needed to compute several cashflows (such as incentives repartition) and energy flows (such shared energy)
    Generating also the entry month recording, to facilitate the entry fee calculation and user entries statistics"""

    print(blue("\nGenerating Membership Matrix:"))

    config = yaml.safe_load(open("config.yml", 'r'))
    users_set = yaml.safe_load(open(config["filename_registry_users_yml"], 'r'))

    df_membership = get_monthly_calendar().set_index("month_number") 
    df_entry = get_monthly_calendar().set_index("month_number")

    for user in users_set:

        entry_month = users_set[user]["entry_month"]
        exit_month = users_set[user]["exit_month"]

        if exit_month == "end":
            exit_month = df_membership.index[-1] + 1
        
        df_entry[user] = 0
        df_entry.loc[entry_month,user] = 1

        df_membership[user] = [1*((month >= entry_month) and (month < exit_month)) for month in df_membership.index ] # for each month, 1 meaning plant is operative (thus opex is applicable); 0 means not operative

    df_membership.T.to_csv(config["filename_membership_matrix"])
    df_entry.T.to_csv(config["filename_user_entry_matrix"])

    # getting which users are present in month 1, to bear the constitution costs
    df_t = df_membership.T
    users_present_month_1 = list(df_t[df_t[1] == 1].index)
    add_to_recap_yml("users_present_month_1", users_present_month_1)

    print("\n **** Membership Matrix created! ****")

################################################################################################################################

#

# we add to "user CACER.xlsx" file all parameter for CER and noCER users that are listed in config["filename_data"] file
# def modify_user_CACER_xlsx():

#     global num_rows_cacer_file_bkp # we save this parameter in way to restore the "user CACER.xlsx" file after the creation of all yaml files
    
#     config = yaml.safe_load(open("config.yml", 'r'))
#     filename_users_CACER_xls = config['filename_users_CACER_xls'] # si importa il nome del file .xls contenente gli utenti da simulare (tutte le tipologie di utenza!)

#     # si apre il file excel e si importa il foglio "utenti" in un df
#     # app = xw.App(visible = False)
#     wb = xw.Book(filename_users_CACER_xls)
#     user_CACER_df = wb.sheets["Utenti"].range('A1').options(pd.DataFrame, 
#                                                 header=1,
#                                                 index=False, 
#                                                 expand='table').value

#     num_rows_cacer_file = len(user_CACER_df.index)  
#     num_rows_cacer_file_bkp = num_rows_cacer_file # we save this parameter in way to restore the "user CACER.xlsx" file after the creation of all yaml files

#     wb.sheets["Utenti"].range("B2:B" + str(num_rows_cacer_file + 1)).clear_contents() # clearing contents for the column "flag", we considered the real number of rows
#     wb.sheets["Utenti"].range("D2:D" + str(num_rows_cacer_file + 1)).clear_contents() # clearing contents for the column "num", we considered the real number of rows

#     #######################################################################################################################################################################

#     filename = config["filename_data"]
#     load_df = pd.read_excel(filename, sheet_name="load")
#     n = load_df.columns.get_loc('flag')
#     load_df = load_df.iloc[:, n:]

#     num_rows = len(load_df.index) # number of load to import inside the "user CACER.xlsx" file

#     range = "A" + str(num_rows_cacer_file + 2) + ":A" + str(num_rows_cacer_file + 1 + num_rows) # define range for pasting formula
#     formula = wb.sheets["Utenti"].range("A2").formula # copy formula to paste
#     wb.sheets["Utenti"].range(range).formula = formula # paste in the first column the formula to estimate the id number of user inside "user CACER.xlsx" file

#     range = "B" + str(num_rows_cacer_file + 2) # define range to paste load parameters
#     wb.sheets["Utenti"][range].options(pd.DataFrame, header=0, index=False, expand='table').value = load_df # paste load parameters

#     wb.save()
#     wb.close()
#     app.quit()

#     print("All users added to user CACER xlsx file!")

################################################################################################################################

# 

# def restore_user_CACER_xlsx():
#     config = yaml.safe_load(open("config.yml", 'r'))
#     filename_users_CACER_xls = config['filename_users_CACER_xls'] # si importa il nome del file .xls contenente gli utenti da simulare (tutte le tipologie di utenza!)

#     # si apre il file excel e si importa il foglio "utenti" in un df
#     app = xw.App(visible = False)
#     wb = xw.Book(filename_users_CACER_xls)
#     user_CACER_df = wb.sheets["Utenti"].range('A1').options(pd.DataFrame, 
#                                                         header=1,
#                                                         index=False, 
#                                                         expand='table').value
    
#     num_rows = len(user_CACER_df)
    
#     range = str(num_rows_cacer_file_bkp + 2) + ":" + str(num_rows + 1) # define range of values to clean
#     wb.sheets["Utenti"].range(range).api.Delete(DeleteShiftDirection.xlShiftUp) # clean values in excel file

#     wb.save()
#     wb.close()
#     app.quit()

#     print("User CACER xlsx file restored!")

################################################################################################################################

def copy_folder_content(source_folder, destination_folder):
    """copy and paste of folder content from source_folder to destination_folder"""
    
    # create the destination folder if it doesn't exist
    os.makedirs(destination_folder, exist_ok=True)

    # check if the source folder is a file or a folder
    if os.path.isfile(source_folder):
        # copy the file to the destination folder
        destination_path = os.path.join(destination_folder, os.path.basename(source_folder))
        shutil.copy2(source_folder, destination_path)
    elif os.path.isdir(source_folder):
        # iterate over the files and subfolders in the source folder
        for filename in os.listdir(source_folder):
            # get the full path of the file
            source_path = os.path.join(source_folder, filename)
            destination_path = os.path.join(destination_folder, filename)

            # check if it's a file or a directory
            if os.path.isfile(source_path):
                # copy the file to the destination folder
                shutil.copy2(source_path, destination_path)
            elif os.path.isdir(source_path):
                # recursively copy the subfolder
                copy_folder_content(source_path, destination_path)

def save_simulation_results(simulation_name="test"):
    """Saving the main simulation results in a folder with the simulation_name, for the record"""
    
    print(blue("\nSave all finance results:", ['bold', 'underlined']), '\n')

    config = yaml.safe_load(open("config.yml", 'r'))
    recap = yaml.safe_load(open(config["filename_recap"], 'r'))
    destination_folder = config["foldername_result_finance"] + "\\" + simulation_name
    # clear_folder_content(destination_folder)

    copy_folder_content(config["foldername_finance"], destination_folder)
    copy_folder_content(config["filename_CACER_energy_monthly"], destination_folder)
    copy_folder_content(config["filename_CACER_incentivi"], destination_folder)
    copy_folder_content(config["filename_membership_matrix"], destination_folder)
    copy_folder_content(config["filename_plant_operation_matrix"], destination_folder)
    copy_folder_content(config["filename_input_FM_excel"], destination_folder)
    copy_folder_content(config["filename_registry_plants_yml"], destination_folder)
    copy_folder_content(config["filename_registry_user_types_yml"], destination_folder)
    copy_folder_content(config["filename_registry_users_yml"], destination_folder)
    copy_folder_content(config["filename_FM_results_last_simulation"], destination_folder)
    copy_folder_content(config["filename_recap"], destination_folder)
    output_file_docx = config["foldername_result_finance"] + recap["case_denomination"] + '.docx'
    copy_folder_content(output_file_docx, destination_folder)

    print("\n**** All finance results saved! ****")

################################################################################################################################

def kill_excel_processes():

    print(blue("\nKilling all excel process in backgroud:", ['bold', 'underlined']), '\n')

    for proc in psutil.process_iter(['pid', 'name']):
        if proc.info['name'] and 'excel' in proc.info['name'].lower():
            try:
                os.kill(proc.info['pid'], signal.SIGTERM)
                # print(f"Killed Excel process: {proc.info['pid']}")
            except Exception as e:
                print(f"Could not kill Excel process {proc.info['pid']}: {e}")
    
    print("\n**** All excel processes killed! ****")

################################################################################################################################

def setting_CACER_scenario(case_denomination, CACER_type, repartition_scheme):
    """Setting the CACER scenario.
    Args:
        case_denomination (string): denomination of the case
        CACER_type (string): type of CACER
        repartition_scheme (string): repartition scheme for incentives
    """

    print(blue("\nSetting input financial simulation:", ['bold', 'underlined']), '\n')

    add_to_recap_yml(key = "case_denomination", value = case_denomination)
    add_to_recap_yml(key = "type_of_cacer", value = CACER_type)
    edit_incentive_repartition_scheme(repartition_scheme)
    print("**** All inputs setted! ****")
