"""File 01-descriptive.py

Michel Bierlaire
Sat Aug 02 2025, 16:32:08
"""

# Before using a data file for modeling purposes, it is important to
# collect some information about its content. The objective of this lab is to extract some descriptive statistics
# from a database with choice data using the package `pandas`.

# We introduce some examples using the file `swissmetro.dat`. 

# We first import `pandas`
import pandas as pd
from IPython.core.display_functions import display
from matplotlib import pyplot as plt
from biogeme.data.swissmetro import read_data

# The data file is available at
# [http://transp-or.epfl.ch/data/swissmetro.dat](http://transp-or.epfl.ch/data/swissmetro.dat).

# The
# description of the columns of the file is
# available [here](http://transp-or.epfl.ch/documents/technicalReports/CS_SwissmetroDescription.pdf).

# Read the data. It is provided with the Biogeme distribution.
swissmetro = read_data()
display(swissmetro.dataframe)


# - The database contains 10728 rows of data, corresponding to each
# observation in the sample.
# - It contains 28 columns, corresponding to
# the available variables.

# The list of columns is reported below.
display(swissmetro.dataframe.columns)

# If we look at the column `ID`, we observe that it contains 1192
# unique values, corresponding to the 1192 individuals that have
# participated in the survey. Each of these respondents was asked to
# perform 9 choice exercises, for a total of 10728 observations (the
# number of rows in the file).

display(swissmetro.dataframe['ID'].unique())

# If we look at the column `PURPOSE`, corresponding to the trip
# purpose, it contains a total of 9 unique values, numbered from 1 to
# 9. 
display(swissmetro.dataframe['PURPOSE'].unique())

# In order to understand better the distribution of these values, we
# can calculate the frequency of each value, here sorted by decreasing
# order of frequency.
display(swissmetro.dataframe['PURPOSE'].value_counts())

# The histogram of this distribution is also useful.
_ = swissmetro.dataframe['PURPOSE'].value_counts().plot(title='PURPOSE', kind='bar')
plt.show()

# We do the same for the `CHOICE`variable.
display(swissmetro.dataframe['CHOICE'].value_counts())

# And the histogram...
_ = swissmetro.dataframe['CHOICE'].value_counts().plot(title='CHOICE', kind='bar')
plt.show()

#  If we look at the column `INCOME`, we note that it is also
# coded as a discrete variables, with 5 unique values, distributed as follows.
swissmetro.dataframe['INCOME'].value_counts()


# And we can represent the histogram using horizontal bars. 
_ = swissmetro.dataframe['INCOME'].value_counts().plot(title='INCOME', kind='barh')
plt.show()

# If we look at a continuous variable, such as `TRAIN_TT`,
# representing the travel time by train, we are interested in statistics
# such as the mean, the standard deviation, the minimum and maximum
# values, as well as some quantiles.

display(swissmetro.dataframe['TRAIN_TT'].describe())


# It is interesting to note that 75\% of the values are lesser or equal
# to 209, while the maximum is 1049. 

# A histogram can also be plotted.
_ = swissmetro.dataframe['TRAIN_TT'].hist()


#  A similar analysis of the variable `SM_CO` provides the
# following statistics.
display(swissmetro.dataframe['SM_CO'].describe())
_ = swissmetro.dataframe['SM_CO'].hist()
plt.show()

# It may be made more readable by using a log scale.
_ = swissmetro.dataframe['SM_CO'].hist(log=True)
plt.show()

# It is also interesting to investigate the correlation between
# two variables.
display(swissmetro.dataframe['TRAIN_TT'].corr(swissmetro.dataframe['TRAIN_CO']))


# The correlation can also be illustrated using a scatter plot.
_ = swissmetro.dataframe.plot(kind='scatter', x='TRAIN_TT', y='TRAIN_CO', color='r')
plt.show()

# Now, you are asked to perform a similar analysis of the file
# [http://transp-or.epfl.ch/data/optima.dat](http://transp-or.epfl.ch/data/optima.dat). The description of the data is
# available [here](http://transp-or.epfl.ch/documents/technicalReports/CS_OptimaDescription.pdf).
# The file can also be obtained from the Biogeme distribution using `from biogeme.data.optima import read_data`