"""File 01-descriptive_solution.py

Michel Bierlaire
Sat Aug 02 2025, 16:38:03
"""

import pandas as pd
from IPython.core.display_functions import display
from matplotlib import pyplot as plt
from biogeme.data.optima import read_data

# You are asked to perform a descriptive analysis of the file
# [http://transp-or.epfl.ch/data/optima.dat](http://transp-or.epfl.ch/data/optima.dat). The description of the data is
# available [here](http://transp-or.epfl.ch/documents/technicalReports/CS_OptimaDescription.pdf).
optima = read_data()
display(optima.dataframe)


# We first note that there are 2265 observations, but only 1763 IDs. The reason is that some respondents have reported
# several tours for the day of reference.
display(len(optima.dataframe['ID'].unique()))


# As there are many columns, we use a loop to enumerate them.
for c in optima.dataframe.columns:
    print(c)


# We first investigate the discrete variable `Education`.
display(optima.dataframe['Education'].value_counts())

_ = optima.dataframe['Education'].value_counts().plot(kind='bar')
plt.show()

# We also investigate a continuous variable `TimePT`.
display(optima.dataframe['TimePT'].describe())


# And the corresponding histogram.
_ = optima.dataframe['TimePT'].hist(log=True)
plt.show()

# We note that income is available in two columns: `Income` and `CalculatedIncome`.
display(optima.dataframe['Income'].value_counts())
display(optima.dataframe['CalculatedIncome'].value_counts())

# And we can plot the two variables.
_ = optima.dataframe.plot(kind='scatter', x='Income', y='CalculatedIncome', color='r')
plt.show()

# When income was reported, the calculated income is simply the center of the corresponding interval.
# For instance, `Income = 2` corresponds to a monthly income from 2501 to 4000 CHF. In that case,
# `CalculatedIncome = 3250`. When income was not reported (`Income = -1`), the calculated income has been inputted.
# The method used for imputation is not reported in the documentation.


# It can be noted that the travel time by car and the distance in kilometers are strongly correlated. 
display(optima.dataframe['TimeCar'].corr(optima.dataframe['distance_km']))

# And we can plot the two variables.
_ = optima.dataframe.plot(kind='scatter', x='distance_km', y='TimeCar', color='r')
plt.show()

# The travel time by public transportation and distance are also correlated, although less so.
display(optima.dataframe['TimePT'].corr(optima.dataframe['distance_km']))

# And we can plot the two variables.
_ = optima.dataframe.plot(kind='scatter', x='distance_km', y='TimePT', color='r')
plt.show()