dnnlinearregressor.py

# -*- coding: utf-8 -*-
"""DNNLinearRegressor.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1jeKBL0D5yBPta4ED-rFzZcH0SesKCa_q
"""

# import necessary packages
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
import tensorflow as tf
import itertools
import numpy as np

# Set ipython's max row display
pd.set_option('display.max_row', 1000)

# Set iPython's max column width to 50
pd.set_option('display.max_columns', 50)

# define the columns to included in the dataframe
COLUMNS = ['crim','zn','indus','chas','nox','rm',
         'age','dis','rad','tax','ptratio','black','lstat','medv']

url='https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data'
# create the dataframe
df = pd.read_csv(url, delim_whitespace=True, names=COLUMNS)

# shuffle the data and reset the index
df.sample(frac=1).reset_index(drop=True)

# print the first 5 rows of the dataframe
print(df.head())

# get the information regarding the dataframe
print(df.info())

# get the statistics of the data 
print(df.describe().transpose())

# plot the boxplot using seaborn
sns.set(style='ticks')
sns.boxplot(data=df)

# plot the pairplot of the dataframe
sns.pairplot(data=df)

# compute the correlation among the features in the dataset
corr_data = df.corr(method='pearson')
print(corr_data)

# plot the correlation matrix data using matplotlib
plt.matshow(corr_data)
plt.xticks(range(len(corr_data.columns)), corr_data.columns)
plt.yticks(range(len(corr_data.columns)), corr_data.columns)
plt.colorbar()
plt.show()

# define the columns to get scaled except the 'age' column and the dependent column 'medv' 
df_subcategories = df.loc[:, ~df.columns.isin(['age', 'medv'])]

# instantiate contructor for the MinMaxScaler for the independent variables
x_scaler = MinMaxScaler()
#fit and transform the data except for 'age' and 'medv' columns
df_scaled = x_scaler.fit_transform(df_subcategories)
df.loc[:, ~df.columns.isin([ 'age', 'medv'])] = df_scaled

# instantiate contructor for the MinMaxScaler for the dependent variable 'medv'
y_scaler = MinMaxScaler()
# fit and transform the 'medv' column
df_y_scaled = y_scaler.fit_transform(df.loc[:, 'medv'].values.reshape(-1,1))
df.loc[:, 'medv'] = df_y_scaled
print(df)

# create a list of continuous value columns to parsed for feature columns except 'age' and 'medv'. 'age' can be parsed into bucketized column and 'medv' shouldn't be converted into a feature column since it is the dependent column
continuous_columns = [c for c in df.columns if c not in ['age',  'medv']]
bucketized_columns = ['age']

# convert the continous columns into numeric feature columns
cont_feature_cols = [tf.feature_column.numeric_column(c) for c in continuous_columns]
print(cont_feature_cols)

# first convert the 'age' column into numeric feature column and then into a bucketized feature column with the boundaries set in the 'boundaries' parameter 
age_numeric_cols = tf.feature_column.numeric_column('age')
age_bucketized_cols = [tf.feature_column.bucketized_column(source_column=age_numeric_cols, boundaries=[i for i in range(0, 105, 5)])]
print(age_bucketized_cols)

print(age_bucketized_cols)

# instantiate estimator dnn linear regressor model with the feature columns defined earlier and the number of neurons on each hidden layer by passing on a list
estimator = tf.estimator.DNNLinearCombinedRegressor(linear_feature_columns=age_bucketized_cols,
                                                    dnn_feature_columns=cont_feature_cols,
                                                    dnn_hidden_units=[2056, 1024, 512, 256, 128], 
                                                    model_dir='DNN_Linear_train_1')

# split the dataset into train and test sets
train_data = df.iloc[:int(0.8*len(df)), :]
test_data = df.iloc[int(0.8*len(df)):,:]
print(train_data.shape, test_data.shape)
print(df.head())

# convert the dataframes into csv files to be passed into the tensorflow dataset functions
train_data_csv = train_data.to_csv('train_data.csv', index=None, header=True)
test_data_csv = test_data.to_csv('test_data.csv', index=None, header=True)

trains = 'train_data.csv'
tests = 'test_data.csv'

# define a list having the length of the columns with zeros 
record_default = [[0.0] for i in range(len(COLUMNS))]

# build the input function to be used for training, evaulation and prediction
def input_fn(csv_data, batch_size, num_epochs=None):
  # define a parser function to be applied to the later
  def parse_csv(value):
    # decode the csv file using tf.decode_csv  
    columns = tf.decode_csv(value, record_defaults=record_default)
    # get the features and the label
    features = dict(zip(COLUMNS, columns))
    labels = features.pop('medv')
    return features, labels
  
  # use the TextLineDataset function of tensorflow to read off csv
  dataset = tf.data.TextLineDataset(csv_data)
  # skip the header
  dataset = dataset.skip(1)
  # map the dataset using the parser function defined
  dataset = dataset.map(parse_csv)
  # allow the dataset to continue indefinetely to feed the model 
  dataset = dataset.repeat(num_epochs)
  # define the batch size of the dataset 
  dataset = dataset.batch(batch_size)
  
  # create an iterator to iterate over the dataset
  iterator = dataset.make_one_shot_iterator()
  # get the next batch of data off dataset
  features, labels = iterator.get_next()
  return features, labels

# train the estimator model
estimator.train(steps=1000, input_fn=lambda: input_fn(trains, batch_size=128, num_epochs=100))

# evaluate the estimator model
ev = estimator.evaluate(steps=None, input_fn=lambda: input_fn(tests, batch_size=128, num_epochs=1))

# get the evaluation loss
loss = ev['loss']
print('Loss: ', loss)

# build the prediction input data
prediction_input = {				
          'crim': [0.03359,5.09017,0.12650,0.05515,8.15174,0.24522],				
          'zn': [75.0,0.0,25.0,33.0,0.0,0.0],				
          'indus': [2.95,18.10,5.13,2.18,18.10,9.90],	
          'chas': [0, 0, 1, 0, 0, 1],
          'nox': [0.428,0.713,0.453,0.472,0.700,0.544],				
          'rm': [7.024,6.297,6.762,7.236,5.390,5.782],				
          'age': [15.8,91.8,43.4,41.1,98.9,71.7],				
          'dis': [5.4011,2.3682,7.9809,4.0220,1.7281,4.0317],	
          'rad': [1,2,3,1,3,2],
          'tax': [252,666,284,222,666,304],				
          'ptratio': [18.3,20.2,19.7,18.4,20.2,18.4],
          'black': [396.2, 288.34, 393.5, 299.9, 394.6, 395.3],
          'lstat': [9, 10, 11, 12, 14, 15]
     }

# convert prediction input data into a dataframe for rescaling the values
prediction_dataframe = pd.DataFrame(prediction_input)
# rescale the values to be fed into the model for prediction
prediction_dataframe.loc[:, ~prediction_dataframe.columns.isin(['age'])] = x_scaler.transform(prediction_dataframe.loc[:, ~prediction_dataframe.columns.isin(['age'])])
print(prediction_dataframe)
# convert the prediction dataframe to a dictionary as the prediction function of tensorflow dataset expects dictionary
prediction_dict = prediction_dataframe.to_dict(orient='list')
print(prediction_dict)

def prediction_fn():
  # gather the data by using the from_tensors function 
  dataset = tf.data.Dataset.from_tensors(prediction_dict)
  return dataset

# predict the result
prediction_results = estimator.predict(input_fn=prediction_fn)

# iterate over the predicted result and do inverse rescale transform to get the original predicted data
for prediction in enumerate(prediction_results):
  print(y_scaler.inverse_transform((prediction[1]['predictions'].reshape(-1, 1))))