9  Tree-Based Models for Regression - Predicting a Numeric Indicator of Diabetes Disease Progression

# Original Code source: Jaques Grobler: https://scikit-learn.org/stable/auto_examples/linear_model/plot_ols.html
# License: BSD 3 clause
# Modified by SR

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from sklearn import datasets
from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error, \
                            r2_score, root_mean_squared_error


from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression

from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from sklearn.ensemble import HistGradientBoostingRegressor

9.1 The dataset

https://scikit-learn.org/stable/datasets/toy_dataset.html#diabetes-dataset

Ten baseline variables, age, sex, body mass index, average blood pressure, and six blood serum measurements were obtained for each of n = 442 diabetes patients, as well as the response of interest, a quantitative measure of disease progression one year after baseline.

# Load the diabetes dataset
diabetes_X, diabetes_y = datasets.load_diabetes(return_X_y=True, as_frame=True, scaled=False)

diabetes_X.head()

	age	sex	bmi	bp	s1	s2	s3	s4	s5	s6
0	59.0	2.0	32.1	101.0	157.0	93.2	38.0	4.0	4.8598	87.0
1	48.0	1.0	21.6	87.0	183.0	103.2	70.0	3.0	3.8918	69.0
2	72.0	2.0	30.5	93.0	156.0	93.6	41.0	4.0	4.6728	85.0
3	24.0	1.0	25.3	84.0	198.0	131.4	40.0	5.0	4.8903	89.0
4	50.0	1.0	23.0	101.0	192.0	125.4	52.0	4.0	4.2905	80.0

diabetes_y.head()

0    151.0
1     75.0
2    141.0
3    206.0
4    135.0
Name: target, dtype: float64

# Split the data into training/testing sets
diabetes_X_train, diabetes_X_test, diabetes_y_train, diabetes_y_test = train_test_split(
    diabetes_X, diabetes_y,
    test_size = 0.25,
    random_state=42
    )

diabetes_X_train

	age	sex	bmi	bp	s1	s2	s3	s4	s5	s6
16	47.0	1.0	30.3	109.0	207.0	100.2	70.0	3.00	5.2149	98.0
408	66.0	1.0	21.7	126.0	212.0	127.8	45.0	4.71	5.2781	101.0
432	51.0	1.0	31.5	93.0	231.0	144.0	49.0	4.70	5.2523	117.0
316	53.0	2.0	27.7	95.0	190.0	101.8	41.0	5.00	5.4638	101.0
3	24.0	1.0	25.3	84.0	198.0	131.4	40.0	5.00	4.8903	89.0
...	...	...	...	...	...	...	...	...	...	...
106	22.0	1.0	19.3	82.0	156.0	93.2	52.0	3.00	3.9890	71.0
270	50.0	2.0	29.2	119.0	162.0	85.2	54.0	3.00	4.7362	95.0
348	57.0	1.0	24.5	93.0	186.0	96.6	71.0	3.00	4.5218	91.0
435	45.0	1.0	24.2	83.0	177.0	118.4	45.0	4.00	4.2195	82.0
102	23.0	1.0	29.0	90.0	216.0	131.4	65.0	3.00	4.5850	91.0

331 rows × 10 columns

diabetes_y_train

16     166.0
408    189.0
432    173.0
316    220.0
3      206.0
       ...  
106    134.0
270    202.0
348    148.0
435     64.0
102    302.0
Name: target, Length: 331, dtype: float64

# Explore distribution of values being predicted
diabetes_y_train.hist()

pd.DataFrame(diabetes_y_train).plot(kind="box")

9.2 Storing results

Create a list to store results in.

results_list = []

9.3 Linear Regression

def standardise_data(X_train, X_test):

    # Initialise a new scaling object for normalising input data
    sc = StandardScaler()

    # Apply the scaler to the training and test sets
    train_std=sc.fit_transform(X_train)
    test_std=sc.fit_transform(X_test)

    return train_std, test_std

diabetes_X_train_standardised, diabetes_X_test_standardised = standardise_data(
    diabetes_X_train,
    diabetes_X_test
    )

# Use only first feature
single_feat_train = diabetes_X_train_standardised[:, np.newaxis, 2]
single_feat_test = diabetes_X_test_standardised[:, np.newaxis, 2]

# Create linear regression object
regr = LinearRegression()

# Train the model using the training sets
regr.fit(single_feat_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr.predict(single_feat_test)

print(f"Mean absolute error: {mean_absolute_error(diabetes_y_test, diabetes_y_pred):.2f}")

print(f"Mean absolute percentage error: {mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred):.2%}" )

print("Root Mean squared error: %.2f" % root_mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

results_list.append(
    {'model': 'Linear Regression',
     'RMSE': root_mean_squared_error(diabetes_y_test, diabetes_y_pred),
     'MAPE': mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred)}
)

# Plot outputs
def plot_actual_vs_predicted(actual, predicted):
    fig, ax = plt.subplots(figsize=(6, 6))

    ax.scatter(actual, predicted, color="black")
    ax.axline((1, 1), slope=1)
    plt.xlabel('True Values')
    plt.ylabel('Predicted Values')
    plt.title('True vs Predicted Values')
    plt.show()

def plot_residuals(actual, predicted):
    residuals = actual - predicted

    plt.figure(figsize=(10, 5))
    plt.hist(residuals, bins=20)
    plt.axvline(x = 0, color = 'r')
    plt.xlabel('Residual')
    plt.ylabel('Frequency')
    plt.title('Distribution of Residuals')
    plt.show()

plot_actual_vs_predicted(diabetes_y_test, diabetes_y_pred)

Mean absolute error: 51.02
Mean absolute percentage error: 48.88%
Root Mean squared error: 62.00
Coefficient of determination: 0.30

plot_residuals(diabetes_y_test, diabetes_y_pred)

9.4 Multiple Linear Regression

# Create linear regression object
regr = LinearRegression()

# Train the model using the training sets
regr.fit(diabetes_X_train_standardised, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr.predict(diabetes_X_test_standardised)

print(f"Mean absolute error: {mean_absolute_error(diabetes_y_test, diabetes_y_pred):.2f}")

print(f"Mean absolute percentage error: {mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred):.2%}" )

print("Root Mean squared error: %.2f" % root_mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

results_list.append(
    {'model': 'Multiple Linear Regression',
     'RMSE': root_mean_squared_error(diabetes_y_test, diabetes_y_pred),
     'MAPE': mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred)}
)

plot_actual_vs_predicted(diabetes_y_test, diabetes_y_pred)

Mean absolute error: 42.42
Mean absolute percentage error: 40.74%
Root Mean squared error: 54.25
Coefficient of determination: 0.47

plot_residuals(diabetes_y_test, diabetes_y_pred)

9.5 Decision Tree

regr_dt = DecisionTreeRegressor(random_state=42)

# Train the model using the training sets
regr_dt.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr_dt.predict(diabetes_X_test)

print(f"Mean absolute error: {mean_absolute_error(diabetes_y_test, diabetes_y_pred):.2f}")


print(f"Mean absolute percentage error: {mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred):.2%}" )

print("Root Mean squared error: %.2f" % root_mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

results_list.append(
    {'model': 'Decision Tree',
     'RMSE': root_mean_squared_error(diabetes_y_test, diabetes_y_pred),
     'MAPE': mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred)}
)

plot_actual_vs_predicted(diabetes_y_test, diabetes_y_pred)

Mean absolute error: 57.55
Mean absolute percentage error: 50.68%
Root Mean squared error: 76.00
Coefficient of determination: -0.04

plot_residuals(diabetes_y_test, diabetes_y_pred)

9.6 Random Forest

regr_rf = RandomForestRegressor(random_state=42)

# Train the model using the training sets
regr_rf.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr_rf.predict(diabetes_X_test)

print(f"Mean absolute error: {mean_absolute_error(diabetes_y_test, diabetes_y_pred):.2f}")


print(f"Mean absolute percentage error: {mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred):.2%}" )

print("Root Mean squared error: %.2f" % root_mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

results_list.append(
    {'model': 'Random Forest',
     'RMSE': root_mean_squared_error(diabetes_y_test, diabetes_y_pred),
     'MAPE': mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred)}
)

plot_actual_vs_predicted(diabetes_y_test, diabetes_y_pred)

Mean absolute error: 43.86
Mean absolute percentage error: 40.34%
Root Mean squared error: 54.96
Coefficient of determination: 0.45

plot_residuals(diabetes_y_test, diabetes_y_pred)

9.7 XGBoost

regr_xg = XGBRegressor(random_state=42)

# Train the model using the training sets
regr_xg.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr_xg.predict(diabetes_X_test)

print(f"Mean absolute error: {mean_absolute_error(diabetes_y_test, diabetes_y_pred):.2f}")


print(f"Mean absolute percentage error: {mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred):.2%}" )

print("Root Mean squared error: %.2f" % root_mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

results_list.append(
    {'model': 'XGBoost',
     'RMSE': root_mean_squared_error(diabetes_y_test, diabetes_y_pred),
     'MAPE': mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred)}
)

plot_actual_vs_predicted(diabetes_y_test, diabetes_y_pred)

Mean absolute error: 46.59
Mean absolute percentage error: 39.84%
Root Mean squared error: 58.01
Coefficient of determination: 0.39

plot_residuals(diabetes_y_test, diabetes_y_pred)

9.8 Light GBM

regr_lgbm = LGBMRegressor(random_state=42)

# Train the model using the training sets
regr_lgbm.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr_lgbm.predict(diabetes_X_test)

print(f"Mean absolute error: {mean_absolute_error(diabetes_y_test, diabetes_y_pred):.2f}")


print(f"Mean absolute percentage error: {mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred):.2%}" )

print("Root Mean squared error: %.2f" % root_mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

results_list.append(
    {'model': 'Light GBM',
     'RMSE': root_mean_squared_error(diabetes_y_test, diabetes_y_pred),
     'MAPE': mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred)}
)

plot_actual_vs_predicted(diabetes_y_test, diabetes_y_pred)

Mean absolute error: 45.91
Mean absolute percentage error: 40.71%
Root Mean squared error: 57.97
Coefficient of determination: 0.39

plot_residuals(diabetes_y_test, diabetes_y_pred)

9.9 Catboost

regr_catboost = CatBoostRegressor(random_state=42)

# Train the model using the training sets
regr_catboost.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr_catboost.predict(diabetes_X_test)

print(f"Mean absolute error: {mean_absolute_error(diabetes_y_test, diabetes_y_pred):.2f}")


print(f"Mean absolute percentage error: {mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred):.2%}" )

print("Root Mean squared error: %.2f" % root_mean_squared_error(diabetes_y_test, diabetes_y_pred))
# The coefficient of determination: 1 is perfect prediction
print("Coefficient of determination: %.2f" % r2_score(diabetes_y_test, diabetes_y_pred))

results_list.append(
    {'model': 'CatBoost',
     'RMSE': root_mean_squared_error(diabetes_y_test, diabetes_y_pred),
     'MAPE': mean_absolute_percentage_error(diabetes_y_test, diabetes_y_pred)}
)

plot_actual_vs_predicted(diabetes_y_test, diabetes_y_pred)

Learning rate set to 0.034381
0:  learn: 76.7698931   total: 1.5ms    remaining: 1.5s
1:  learn: 75.7552747   total: 2.5ms    remaining: 1.25s
2:  learn: 74.8136416   total: 3.54ms   remaining: 1.18s
3:  learn: 73.9549588   total: 4.85ms   remaining: 1.21s
4:  learn: 73.0782030   total: 6.35ms   remaining: 1.26s
5:  learn: 72.1682178   total: 7.69ms   remaining: 1.27s
6:  learn: 71.3682200   total: 8.78ms   remaining: 1.25s
7:  learn: 70.5608211   total: 9.75ms   remaining: 1.21s
8:  learn: 69.7224552   total: 10.9ms   remaining: 1.2s
9:  learn: 68.9543606   total: 12.2ms   remaining: 1.21s
10: learn: 68.2429798   total: 13.5ms   remaining: 1.21s
11: learn: 67.5175992   total: 14.8ms   remaining: 1.22s
12: learn: 66.8114535   total: 16.1ms   remaining: 1.22s
13: learn: 66.1442549   total: 17.3ms   remaining: 1.22s
14: learn: 65.5281514   total: 18.5ms   remaining: 1.22s
15: learn: 64.8807317   total: 19.7ms   remaining: 1.21s
16: learn: 64.3379117   total: 21.1ms   remaining: 1.22s
17: learn: 63.7508639   total: 22.4ms   remaining: 1.22s
18: learn: 63.2097439   total: 23.6ms   remaining: 1.22s
19: learn: 62.7139608   total: 24.7ms   remaining: 1.21s
20: learn: 62.2236234   total: 26ms remaining: 1.21s
21: learn: 61.7820840   total: 27.1ms   remaining: 1.2s
22: learn: 61.2214417   total: 28ms remaining: 1.19s
23: learn: 60.7727226   total: 28.9ms   remaining: 1.18s
24: learn: 60.3932562   total: 29.9ms   remaining: 1.16s
25: learn: 59.9563855   total: 30.8ms   remaining: 1.15s
26: learn: 59.5363233   total: 31.9ms   remaining: 1.15s
27: learn: 59.1791749   total: 33.1ms   remaining: 1.15s
28: learn: 58.6907023   total: 34.1ms   remaining: 1.14s
29: learn: 58.2293662   total: 35ms remaining: 1.13s
30: learn: 57.8130967   total: 36ms remaining: 1.13s
31: learn: 57.4281744   total: 37.3ms   remaining: 1.13s
32: learn: 57.1725832   total: 38.5ms   remaining: 1.13s
33: learn: 56.7632606   total: 39.6ms   remaining: 1.13s
34: learn: 56.4138620   total: 40.7ms   remaining: 1.12s
35: learn: 56.0466281   total: 42ms remaining: 1.12s
36: learn: 55.7211225   total: 43.1ms   remaining: 1.12s
37: learn: 55.4148873   total: 44.2ms   remaining: 1.12s
38: learn: 55.1553516   total: 45.2ms   remaining: 1.11s
39: learn: 54.8464593   total: 46.2ms   remaining: 1.11s
40: learn: 54.5206398   total: 47.3ms   remaining: 1.11s
41: learn: 54.2016506   total: 48.3ms   remaining: 1.1s
42: learn: 53.9583345   total: 49.2ms   remaining: 1.09s
43: learn: 53.6716694   total: 50.7ms   remaining: 1.1s
44: learn: 53.4208321   total: 51.9ms   remaining: 1.1s
45: learn: 53.1852474   total: 53ms remaining: 1.1s
46: learn: 52.8905478   total: 54ms remaining: 1.09s
47: learn: 52.7149197   total: 55ms remaining: 1.09s
48: learn: 52.5124637   total: 56.1ms   remaining: 1.09s
49: learn: 52.3121632   total: 57.1ms   remaining: 1.08s
50: learn: 52.1629426   total: 58ms remaining: 1.08s
51: learn: 51.9156136   total: 59ms remaining: 1.07s
52: learn: 51.6313901   total: 59.8ms   remaining: 1.07s
53: learn: 51.3785559   total: 60.8ms   remaining: 1.06s
54: learn: 51.2121959   total: 61.6ms   remaining: 1.06s
55: learn: 51.0247514   total: 62.4ms   remaining: 1.05s
56: learn: 50.7958459   total: 63.4ms   remaining: 1.05s
57: learn: 50.6322565   total: 64.2ms   remaining: 1.04s
58: learn: 50.4693849   total: 65.1ms   remaining: 1.04s
59: learn: 50.2729021   total: 66.1ms   remaining: 1.03s
60: learn: 50.0652141   total: 67ms remaining: 1.03s
61: learn: 49.9072643   total: 67.9ms   remaining: 1.03s
62: learn: 49.7088928   total: 68.8ms   remaining: 1.02s
63: learn: 49.4991804   total: 69.7ms   remaining: 1.02s
64: learn: 49.3220877   total: 70.5ms   remaining: 1.01s
65: learn: 49.1778938   total: 71.3ms   remaining: 1.01s
66: learn: 49.0257537   total: 72.2ms   remaining: 1s
67: learn: 48.8766285   total: 73ms remaining: 1s
68: learn: 48.7340804   total: 73.8ms   remaining: 996ms
69: learn: 48.6003848   total: 74.6ms   remaining: 991ms
70: learn: 48.4638499   total: 75.4ms   remaining: 986ms
71: learn: 48.3048779   total: 76.2ms   remaining: 982ms
72: learn: 48.1774846   total: 77ms remaining: 978ms
73: learn: 48.0137795   total: 77.8ms   remaining: 973ms
74: learn: 47.8542608   total: 78.7ms   remaining: 971ms
75: learn: 47.6948567   total: 79.7ms   remaining: 969ms
76: learn: 47.5868712   total: 80.7ms   remaining: 967ms
77: learn: 47.4238916   total: 81.8ms   remaining: 966ms
78: learn: 47.2951391   total: 82.9ms   remaining: 966ms
79: learn: 47.1670664   total: 83.8ms   remaining: 963ms
80: learn: 46.9672861   total: 84.7ms   remaining: 961ms
81: learn: 46.8544297   total: 85.6ms   remaining: 958ms
82: learn: 46.7604885   total: 86.6ms   remaining: 956ms
83: learn: 46.5980069   total: 87.4ms   remaining: 953ms
84: learn: 46.4970350   total: 88.2ms   remaining: 950ms
85: learn: 46.3157959   total: 89.1ms   remaining: 947ms
86: learn: 46.1764093   total: 89.9ms   remaining: 943ms
87: learn: 46.0119289   total: 90.8ms   remaining: 941ms
88: learn: 45.9075921   total: 91.7ms   remaining: 938ms
89: learn: 45.7041591   total: 92.7ms   remaining: 937ms
90: learn: 45.6230779   total: 93.6ms   remaining: 935ms
91: learn: 45.4441067   total: 94.5ms   remaining: 933ms
92: learn: 45.3302847   total: 95.5ms   remaining: 931ms
93: learn: 45.2266465   total: 96.5ms   remaining: 930ms
94: learn: 45.1188503   total: 97.5ms   remaining: 929ms
95: learn: 45.0070179   total: 98.5ms   remaining: 928ms
96: learn: 44.9073280   total: 99.5ms   remaining: 926ms
97: learn: 44.8299073   total: 100ms    remaining: 925ms
98: learn: 44.7113215   total: 101ms    remaining: 923ms
99: learn: 44.6274691   total: 102ms    remaining: 921ms
100:    learn: 44.5478216   total: 103ms    remaining: 920ms
101:    learn: 44.4544657   total: 104ms    remaining: 918ms
102:    learn: 44.2905178   total: 105ms    remaining: 915ms
103:    learn: 44.2331215   total: 106ms    remaining: 914ms
104:    learn: 44.1630763   total: 107ms    remaining: 913ms
105:    learn: 44.0799029   total: 108ms    remaining: 910ms
106:    learn: 43.9771810   total: 109ms    remaining: 910ms
107:    learn: 43.8666901   total: 110ms    remaining: 909ms
108:    learn: 43.7729359   total: 111ms    remaining: 907ms
109:    learn: 43.6575975   total: 112ms    remaining: 905ms
110:    learn: 43.5374373   total: 113ms    remaining: 903ms
111:    learn: 43.4217869   total: 114ms    remaining: 901ms
112:    learn: 43.2726354   total: 115ms    remaining: 900ms
113:    learn: 43.1591655   total: 116ms    remaining: 898ms
114:    learn: 43.1050945   total: 117ms    remaining: 897ms
115:    learn: 42.9959354   total: 117ms    remaining: 895ms
116:    learn: 42.9189572   total: 118ms    remaining: 893ms
117:    learn: 42.7879437   total: 119ms    remaining: 891ms
118:    learn: 42.7335824   total: 120ms    remaining: 889ms
119:    learn: 42.6647507   total: 121ms    remaining: 886ms
120:    learn: 42.5460610   total: 122ms    remaining: 884ms
121:    learn: 42.4575359   total: 123ms    remaining: 883ms
122:    learn: 42.3488106   total: 124ms    remaining: 882ms
123:    learn: 42.2746008   total: 125ms    remaining: 881ms
124:    learn: 42.1997445   total: 126ms    remaining: 879ms
125:    learn: 42.0779892   total: 126ms    remaining: 877ms
126:    learn: 41.9835915   total: 128ms    remaining: 876ms
127:    learn: 41.8873330   total: 129ms    remaining: 876ms
128:    learn: 41.8151306   total: 130ms    remaining: 875ms
129:    learn: 41.7505622   total: 131ms    remaining: 874ms
130:    learn: 41.6331956   total: 132ms    remaining: 874ms
131:    learn: 41.5887267   total: 133ms    remaining: 872ms
132:    learn: 41.4726460   total: 134ms    remaining: 871ms
133:    learn: 41.4357049   total: 135ms    remaining: 869ms
134:    learn: 41.3683909   total: 135ms    remaining: 867ms
135:    learn: 41.3529582   total: 136ms    remaining: 862ms
136:    learn: 41.2664667   total: 137ms    remaining: 860ms
137:    learn: 41.1269202   total: 137ms    remaining: 858ms
138:    learn: 41.0353019   total: 138ms    remaining: 857ms
139:    learn: 40.9380830   total: 139ms    remaining: 856ms
140:    learn: 40.8300088   total: 140ms    remaining: 855ms
141:    learn: 40.7284181   total: 141ms    remaining: 853ms
142:    learn: 40.6691379   total: 142ms    remaining: 853ms
143:    learn: 40.5847037   total: 144ms    remaining: 853ms
144:    learn: 40.5070803   total: 145ms    remaining: 853ms
145:    learn: 40.4541409   total: 146ms    remaining: 851ms
146:    learn: 40.3628197   total: 146ms    remaining: 849ms
147:    learn: 40.2947908   total: 147ms    remaining: 847ms
148:    learn: 40.2383656   total: 148ms    remaining: 845ms
149:    learn: 40.1418319   total: 149ms    remaining: 844ms
150:    learn: 40.0635154   total: 150ms    remaining: 843ms
151:    learn: 40.0195394   total: 151ms    remaining: 841ms
152:    learn: 39.9984043   total: 152ms    remaining: 841ms
153:    learn: 39.9361859   total: 153ms    remaining: 840ms
154:    learn: 39.8484358   total: 154ms    remaining: 838ms
155:    learn: 39.7381342   total: 155ms    remaining: 836ms
156:    learn: 39.6576067   total: 155ms    remaining: 835ms
157:    learn: 39.6234486   total: 156ms    remaining: 833ms
158:    learn: 39.5537998   total: 157ms    remaining: 832ms
159:    learn: 39.4925269   total: 158ms    remaining: 831ms
160:    learn: 39.4231701   total: 159ms    remaining: 830ms
161:    learn: 39.3822348   total: 160ms    remaining: 829ms
162:    learn: 39.3088103   total: 161ms    remaining: 829ms
163:    learn: 39.2205778   total: 162ms    remaining: 828ms
164:    learn: 39.1371691   total: 163ms    remaining: 826ms
165:    learn: 39.1192417   total: 164ms    remaining: 825ms
166:    learn: 39.0395533   total: 165ms    remaining: 823ms
167:    learn: 38.9513121   total: 166ms    remaining: 822ms
168:    learn: 38.9051022   total: 167ms    remaining: 821ms
169:    learn: 38.8698619   total: 168ms    remaining: 821ms
170:    learn: 38.7690720   total: 169ms    remaining: 820ms
171:    learn: 38.7107433   total: 170ms    remaining: 818ms
172:    learn: 38.5816804   total: 171ms    remaining: 817ms
173:    learn: 38.4737286   total: 172ms    remaining: 816ms
174:    learn: 38.4367775   total: 173ms    remaining: 815ms
175:    learn: 38.3782871   total: 174ms    remaining: 814ms
176:    learn: 38.3075423   total: 175ms    remaining: 813ms
177:    learn: 38.2641161   total: 176ms    remaining: 812ms
178:    learn: 38.1671217   total: 177ms    remaining: 811ms
179:    learn: 38.0724997   total: 178ms    remaining: 809ms
180:    learn: 38.0517931   total: 179ms    remaining: 808ms
181:    learn: 37.9349784   total: 179ms    remaining: 807ms
182:    learn: 37.8350153   total: 180ms    remaining: 805ms
183:    learn: 37.7654821   total: 181ms    remaining: 804ms
184:    learn: 37.6858220   total: 182ms    remaining: 803ms
185:    learn: 37.5583218   total: 183ms    remaining: 801ms
186:    learn: 37.4254625   total: 184ms    remaining: 800ms
187:    learn: 37.4015333   total: 185ms    remaining: 798ms
188:    learn: 37.3565400   total: 186ms    remaining: 797ms
189:    learn: 37.2798855   total: 187ms    remaining: 796ms
190:    learn: 37.1737303   total: 188ms    remaining: 794ms
191:    learn: 37.0917386   total: 189ms    remaining: 794ms
192:    learn: 36.9876853   total: 190ms    remaining: 793ms
193:    learn: 36.9347396   total: 191ms    remaining: 792ms
194:    learn: 36.8560969   total: 192ms    remaining: 791ms
195:    learn: 36.7820441   total: 193ms    remaining: 790ms
196:    learn: 36.7382448   total: 193ms    remaining: 788ms
197:    learn: 36.6888071   total: 194ms    remaining: 787ms
198:    learn: 36.6077915   total: 196ms    remaining: 788ms
199:    learn: 36.5194556   total: 197ms    remaining: 788ms
200:    learn: 36.4391372   total: 198ms    remaining: 787ms
201:    learn: 36.3613057   total: 199ms    remaining: 786ms
202:    learn: 36.3233642   total: 200ms    remaining: 785ms
203:    learn: 36.2923866   total: 201ms    remaining: 783ms
204:    learn: 36.1969353   total: 202ms    remaining: 782ms
205:    learn: 36.1045298   total: 202ms    remaining: 780ms
206:    learn: 36.0651219   total: 203ms    remaining: 779ms
207:    learn: 36.0293077   total: 204ms    remaining: 778ms
208:    learn: 35.9293003   total: 205ms    remaining: 777ms
209:    learn: 35.8540193   total: 206ms    remaining: 776ms
210:    learn: 35.7344981   total: 207ms    remaining: 775ms
211:    learn: 35.6745242   total: 208ms    remaining: 774ms
212:    learn: 35.5702305   total: 209ms    remaining: 772ms
213:    learn: 35.4581545   total: 210ms    remaining: 772ms
214:    learn: 35.3961630   total: 211ms    remaining: 771ms
215:    learn: 35.3470567   total: 212ms    remaining: 770ms
216:    learn: 35.3164110   total: 213ms    remaining: 769ms
217:    learn: 35.3022169   total: 214ms    remaining: 768ms
218:    learn: 35.2201926   total: 215ms    remaining: 768ms
219:    learn: 35.1870608   total: 216ms    remaining: 767ms
220:    learn: 35.1522341   total: 217ms    remaining: 766ms
221:    learn: 35.1144408   total: 218ms    remaining: 765ms
222:    learn: 35.0458040   total: 219ms    remaining: 764ms
223:    learn: 35.0073791   total: 220ms    remaining: 763ms
224:    learn: 34.9968960   total: 221ms    remaining: 762ms
225:    learn: 34.9134931   total: 222ms    remaining: 761ms
226:    learn: 34.8522448   total: 223ms    remaining: 761ms
227:    learn: 34.7182768   total: 224ms    remaining: 759ms
228:    learn: 34.7080690   total: 225ms    remaining: 758ms
229:    learn: 34.6158693   total: 226ms    remaining: 757ms
230:    learn: 34.5994554   total: 227ms    remaining: 756ms
231:    learn: 34.5231509   total: 228ms    remaining: 755ms
232:    learn: 34.4521360   total: 229ms    remaining: 753ms
233:    learn: 34.4377034   total: 230ms    remaining: 752ms
234:    learn: 34.4019649   total: 231ms    remaining: 751ms
235:    learn: 34.3384892   total: 231ms    remaining: 749ms
236:    learn: 34.2648958   total: 232ms    remaining: 748ms
237:    learn: 34.1924062   total: 233ms    remaining: 746ms
238:    learn: 34.0597921   total: 234ms    remaining: 745ms
239:    learn: 33.9936793   total: 235ms    remaining: 744ms
240:    learn: 33.9363340   total: 236ms    remaining: 743ms
241:    learn: 33.8403938   total: 237ms    remaining: 743ms
242:    learn: 33.8253175   total: 238ms    remaining: 742ms
243:    learn: 33.7073406   total: 239ms    remaining: 741ms
244:    learn: 33.6338510   total: 240ms    remaining: 739ms
245:    learn: 33.6226077   total: 241ms    remaining: 738ms
246:    learn: 33.5647316   total: 242ms    remaining: 736ms
247:    learn: 33.4532943   total: 242ms    remaining: 735ms
248:    learn: 33.3957704   total: 243ms    remaining: 734ms
249:    learn: 33.3218832   total: 244ms    remaining: 733ms
250:    learn: 33.2823774   total: 245ms    remaining: 732ms
251:    learn: 33.1949668   total: 246ms    remaining: 731ms
252:    learn: 33.1268050   total: 247ms    remaining: 729ms
253:    learn: 33.0295259   total: 248ms    remaining: 728ms
254:    learn: 32.8956135   total: 249ms    remaining: 727ms
255:    learn: 32.8307975   total: 250ms    remaining: 726ms
256:    learn: 32.7931065   total: 251ms    remaining: 725ms
257:    learn: 32.7541666   total: 252ms    remaining: 725ms
258:    learn: 32.6455644   total: 253ms    remaining: 724ms
259:    learn: 32.5994355   total: 254ms    remaining: 723ms
260:    learn: 32.5175453   total: 255ms    remaining: 721ms
261:    learn: 32.4368870   total: 256ms    remaining: 720ms
262:    learn: 32.4094157   total: 256ms    remaining: 718ms
263:    learn: 32.3119991   total: 257ms    remaining: 717ms
264:    learn: 32.2503354   total: 258ms    remaining: 715ms
265:    learn: 32.1916534   total: 259ms    remaining: 714ms
266:    learn: 32.1189335   total: 260ms    remaining: 713ms
267:    learn: 32.0524602   total: 261ms    remaining: 712ms
268:    learn: 31.9945096   total: 261ms    remaining: 710ms
269:    learn: 31.9432231   total: 262ms    remaining: 709ms
270:    learn: 31.8558105   total: 263ms    remaining: 708ms
271:    learn: 31.7245360   total: 264ms    remaining: 707ms
272:    learn: 31.6810521   total: 265ms    remaining: 706ms
273:    learn: 31.5905310   total: 267ms    remaining: 706ms
274:    learn: 31.5814820   total: 268ms    remaining: 706ms
275:    learn: 31.5287706   total: 269ms    remaining: 705ms
276:    learn: 31.3911913   total: 269ms    remaining: 703ms
277:    learn: 31.2843119   total: 270ms    remaining: 702ms
278:    learn: 31.2729218   total: 271ms    remaining: 701ms
279:    learn: 31.2064306   total: 272ms    remaining: 701ms
280:    learn: 31.1236218   total: 273ms    remaining: 699ms
281:    learn: 31.0698615   total: 274ms    remaining: 698ms
282:    learn: 30.9990092   total: 275ms    remaining: 697ms
283:    learn: 30.9401974   total: 276ms    remaining: 695ms
284:    learn: 30.9325154   total: 277ms    remaining: 694ms
285:    learn: 30.8544742   total: 278ms    remaining: 693ms
286:    learn: 30.7972384   total: 279ms    remaining: 692ms
287:    learn: 30.7880988   total: 279ms    remaining: 691ms
288:    learn: 30.7320574   total: 280ms    remaining: 690ms
289:    learn: 30.7239007   total: 281ms    remaining: 689ms
290:    learn: 30.6605866   total: 282ms    remaining: 688ms
291:    learn: 30.6308656   total: 283ms    remaining: 687ms
292:    learn: 30.5739096   total: 284ms    remaining: 686ms
293:    learn: 30.5626222   total: 285ms    remaining: 685ms
294:    learn: 30.5568473   total: 286ms    remaining: 684ms
295:    learn: 30.5015496   total: 287ms    remaining: 683ms
296:    learn: 30.4270317   total: 288ms    remaining: 682ms
297:    learn: 30.3754589   total: 289ms    remaining: 681ms
298:    learn: 30.3378030   total: 290ms    remaining: 680ms
299:    learn: 30.2313208   total: 291ms    remaining: 679ms
300:    learn: 30.1393072   total: 292ms    remaining: 678ms
301:    learn: 30.0527433   total: 293ms    remaining: 676ms
302:    learn: 29.9579479   total: 294ms    remaining: 676ms
303:    learn: 29.9058132   total: 295ms    remaining: 675ms
304:    learn: 29.8985179   total: 296ms    remaining: 674ms
305:    learn: 29.8111192   total: 297ms    remaining: 673ms
306:    learn: 29.7506065   total: 298ms    remaining: 673ms
307:    learn: 29.6532751   total: 299ms    remaining: 672ms
308:    learn: 29.6013337   total: 300ms    remaining: 671ms
309:    learn: 29.5949394   total: 301ms    remaining: 670ms
310:    learn: 29.5370916   total: 302ms    remaining: 668ms
311:    learn: 29.5077830   total: 303ms    remaining: 667ms
312:    learn: 29.4248526   total: 304ms    remaining: 666ms
313:    learn: 29.3805316   total: 305ms    remaining: 666ms
314:    learn: 29.2893527   total: 306ms    remaining: 665ms
315:    learn: 29.2432232   total: 307ms    remaining: 664ms
316:    learn: 29.2373361   total: 308ms    remaining: 663ms
317:    learn: 29.1224803   total: 309ms    remaining: 662ms
318:    learn: 29.0765583   total: 309ms    remaining: 661ms
319:    learn: 28.9848827   total: 310ms    remaining: 660ms
320:    learn: 28.9083749   total: 311ms    remaining: 658ms
321:    learn: 28.8076597   total: 312ms    remaining: 658ms
322:    learn: 28.7360412   total: 313ms    remaining: 657ms
323:    learn: 28.7158107   total: 314ms    remaining: 656ms
324:    learn: 28.6685417   total: 315ms    remaining: 655ms
325:    learn: 28.6477891   total: 316ms    remaining: 654ms
326:    learn: 28.6027510   total: 317ms    remaining: 653ms
327:    learn: 28.5511355   total: 318ms    remaining: 652ms
328:    learn: 28.5440701   total: 319ms    remaining: 651ms
329:    learn: 28.4708110   total: 320ms    remaining: 650ms
330:    learn: 28.4645627   total: 321ms    remaining: 649ms
331:    learn: 28.3704438   total: 322ms    remaining: 648ms
332:    learn: 28.3634241   total: 323ms    remaining: 648ms
333:    learn: 28.3186244   total: 324ms    remaining: 647ms
334:    learn: 28.2613764   total: 325ms    remaining: 646ms
335:    learn: 28.1538932   total: 326ms    remaining: 645ms
336:    learn: 28.1223246   total: 327ms    remaining: 644ms
337:    learn: 28.0715556   total: 328ms    remaining: 643ms
338:    learn: 28.0045095   total: 329ms    remaining: 642ms
339:    learn: 27.9469559   total: 330ms    remaining: 641ms
340:    learn: 27.9075170   total: 331ms    remaining: 640ms
341:    learn: 27.8393856   total: 332ms    remaining: 639ms
342:    learn: 27.8084813   total: 333ms    remaining: 638ms
343:    learn: 27.8006204   total: 334ms    remaining: 636ms
344:    learn: 27.7940121   total: 335ms    remaining: 635ms
345:    learn: 27.7897137   total: 335ms    remaining: 634ms
346:    learn: 27.7276765   total: 336ms    remaining: 633ms
347:    learn: 27.6434422   total: 337ms    remaining: 632ms
348:    learn: 27.6096334   total: 338ms    remaining: 631ms
349:    learn: 27.5187315   total: 339ms    remaining: 630ms
350:    learn: 27.4845217   total: 340ms    remaining: 628ms
351:    learn: 27.4200688   total: 341ms    remaining: 627ms
352:    learn: 27.3478667   total: 342ms    remaining: 626ms
353:    learn: 27.2974769   total: 343ms    remaining: 625ms
354:    learn: 27.2468761   total: 344ms    remaining: 624ms
355:    learn: 27.1848065   total: 344ms    remaining: 623ms
356:    learn: 27.1263009   total: 345ms    remaining: 622ms
357:    learn: 27.0822692   total: 346ms    remaining: 621ms
358:    learn: 27.0691253   total: 347ms    remaining: 620ms
359:    learn: 27.0269038   total: 348ms    remaining: 619ms
360:    learn: 26.9952141   total: 349ms    remaining: 618ms
361:    learn: 26.9890372   total: 350ms    remaining: 617ms
362:    learn: 26.9465768   total: 351ms    remaining: 616ms
363:    learn: 26.9044433   total: 352ms    remaining: 615ms
364:    learn: 26.8682630   total: 353ms    remaining: 614ms
365:    learn: 26.8618548   total: 354ms    remaining: 612ms
366:    learn: 26.8144006   total: 354ms    remaining: 611ms
367:    learn: 26.8093179   total: 355ms    remaining: 610ms
368:    learn: 26.7618810   total: 356ms    remaining: 609ms
369:    learn: 26.7571170   total: 357ms    remaining: 608ms
370:    learn: 26.7043205   total: 358ms    remaining: 607ms
371:    learn: 26.6378026   total: 359ms    remaining: 606ms
372:    learn: 26.5514300   total: 360ms    remaining: 605ms
373:    learn: 26.4460130   total: 361ms    remaining: 604ms
374:    learn: 26.4398424   total: 362ms    remaining: 603ms
375:    learn: 26.3852939   total: 363ms    remaining: 602ms
376:    learn: 26.3344798   total: 364ms    remaining: 601ms
377:    learn: 26.3272707   total: 365ms    remaining: 600ms
378:    learn: 26.2989990   total: 365ms    remaining: 599ms
379:    learn: 26.2476501   total: 366ms    remaining: 598ms
380:    learn: 26.1918108   total: 367ms    remaining: 596ms
381:    learn: 26.1154654   total: 368ms    remaining: 595ms
382:    learn: 26.1087670   total: 369ms    remaining: 594ms
383:    learn: 26.1040716   total: 370ms    remaining: 593ms
384:    learn: 26.0318993   total: 371ms    remaining: 592ms
385:    learn: 25.9751131   total: 372ms    remaining: 591ms
386:    learn: 25.9379526   total: 373ms    remaining: 590ms
387:    learn: 25.9287243   total: 374ms    remaining: 589ms
388:    learn: 25.8751276   total: 375ms    remaining: 588ms
389:    learn: 25.8300439   total: 376ms    remaining: 587ms
390:    learn: 25.7598570   total: 376ms    remaining: 586ms
391:    learn: 25.7332718   total: 377ms    remaining: 585ms
392:    learn: 25.6894335   total: 378ms    remaining: 584ms
393:    learn: 25.6840453   total: 379ms    remaining: 583ms
394:    learn: 25.6672354   total: 380ms    remaining: 582ms
395:    learn: 25.6613645   total: 381ms    remaining: 581ms
396:    learn: 25.6040062   total: 382ms    remaining: 580ms
397:    learn: 25.5713619   total: 383ms    remaining: 579ms
398:    learn: 25.5146095   total: 384ms    remaining: 578ms
399:    learn: 25.4836363   total: 384ms    remaining: 577ms
400:    learn: 25.4311068   total: 385ms    remaining: 576ms
401:    learn: 25.3692198   total: 386ms    remaining: 575ms
402:    learn: 25.3480067   total: 387ms    remaining: 573ms
403:    learn: 25.3052966   total: 388ms    remaining: 572ms
404:    learn: 25.2519111   total: 389ms    remaining: 571ms
405:    learn: 25.2460603   total: 390ms    remaining: 571ms
406:    learn: 25.2144729   total: 391ms    remaining: 570ms
407:    learn: 25.1347374   total: 392ms    remaining: 569ms
408:    learn: 25.0284719   total: 394ms    remaining: 569ms
409:    learn: 24.9752360   total: 395ms    remaining: 568ms
410:    learn: 24.9260888   total: 396ms    remaining: 567ms
411:    learn: 24.8766066   total: 396ms    remaining: 566ms
412:    learn: 24.8719284   total: 397ms    remaining: 565ms
413:    learn: 24.7976067   total: 398ms    remaining: 564ms
414:    learn: 24.7491830   total: 399ms    remaining: 563ms
415:    learn: 24.7333382   total: 400ms    remaining: 561ms
416:    learn: 24.6709770   total: 401ms    remaining: 560ms
417:    learn: 24.6271914   total: 402ms    remaining: 559ms
418:    learn: 24.5975426   total: 403ms    remaining: 558ms
419:    learn: 24.5646781   total: 404ms    remaining: 557ms
420:    learn: 24.5254733   total: 405ms    remaining: 557ms
421:    learn: 24.4870436   total: 406ms    remaining: 556ms
422:    learn: 24.4832883   total: 407ms    remaining: 555ms
423:    learn: 24.4214499   total: 408ms    remaining: 554ms
424:    learn: 24.4178535   total: 409ms    remaining: 553ms
425:    learn: 24.4120127   total: 409ms    remaining: 552ms
426:    learn: 24.3547134   total: 410ms    remaining: 551ms
427:    learn: 24.3406164   total: 411ms    remaining: 550ms
428:    learn: 24.2735838   total: 412ms    remaining: 549ms
429:    learn: 24.1751947   total: 413ms    remaining: 548ms
430:    learn: 24.1377276   total: 414ms    remaining: 547ms
431:    learn: 24.1304914   total: 415ms    remaining: 546ms
432:    learn: 24.1223580   total: 416ms    remaining: 545ms
433:    learn: 24.0860040   total: 417ms    remaining: 544ms
434:    learn: 24.0466889   total: 418ms    remaining: 543ms
435:    learn: 23.9919742   total: 419ms    remaining: 542ms
436:    learn: 23.9457932   total: 420ms    remaining: 541ms
437:    learn: 23.8815083   total: 421ms    remaining: 540ms
438:    learn: 23.8781224   total: 422ms    remaining: 540ms
439:    learn: 23.8259252   total: 423ms    remaining: 539ms
440:    learn: 23.7926954   total: 424ms    remaining: 538ms
441:    learn: 23.7407856   total: 425ms    remaining: 537ms
442:    learn: 23.7320938   total: 426ms    remaining: 536ms
443:    learn: 23.6915647   total: 427ms    remaining: 535ms
444:    learn: 23.6792172   total: 428ms    remaining: 534ms
445:    learn: 23.6210072   total: 429ms    remaining: 533ms
446:    learn: 23.6161297   total: 430ms    remaining: 532ms
447:    learn: 23.5496392   total: 431ms    remaining: 531ms
448:    learn: 23.5328028   total: 432ms    remaining: 530ms
449:    learn: 23.4639952   total: 433ms    remaining: 529ms
450:    learn: 23.4092092   total: 433ms    remaining: 528ms
451:    learn: 23.3452766   total: 434ms    remaining: 527ms
452:    learn: 23.3140723   total: 436ms    remaining: 526ms
453:    learn: 23.2249015   total: 437ms    remaining: 525ms
454:    learn: 23.1744393   total: 438ms    remaining: 524ms
455:    learn: 23.1699584   total: 439ms    remaining: 523ms
456:    learn: 23.1302089   total: 440ms    remaining: 522ms
457:    learn: 23.0294126   total: 440ms    remaining: 521ms
458:    learn: 22.9840724   total: 441ms    remaining: 520ms
459:    learn: 22.9358059   total: 442ms    remaining: 519ms
460:    learn: 22.8968573   total: 443ms    remaining: 518ms
461:    learn: 22.8930427   total: 444ms    remaining: 517ms
462:    learn: 22.8331553   total: 445ms    remaining: 516ms
463:    learn: 22.7810277   total: 446ms    remaining: 515ms
464:    learn: 22.7759205   total: 447ms    remaining: 514ms
465:    learn: 22.7720110   total: 448ms    remaining: 513ms
466:    learn: 22.7024479   total: 449ms    remaining: 512ms
467:    learn: 22.6995010   total: 450ms    remaining: 511ms
468:    learn: 22.6619011   total: 451ms    remaining: 510ms
469:    learn: 22.6081489   total: 452ms    remaining: 510ms
470:    learn: 22.5927501   total: 453ms    remaining: 509ms
471:    learn: 22.5346437   total: 454ms    remaining: 508ms
472:    learn: 22.4779136   total: 455ms    remaining: 507ms
473:    learn: 22.4276706   total: 456ms    remaining: 506ms
474:    learn: 22.4199813   total: 457ms    remaining: 505ms
475:    learn: 22.3876330   total: 457ms    remaining: 504ms
476:    learn: 22.3528581   total: 458ms    remaining: 503ms
477:    learn: 22.3490392   total: 459ms    remaining: 502ms
478:    learn: 22.3031423   total: 460ms    remaining: 501ms
479:    learn: 22.3007944   total: 461ms    remaining: 500ms
480:    learn: 22.2542492   total: 462ms    remaining: 499ms
481:    learn: 22.2209373   total: 463ms    remaining: 498ms
482:    learn: 22.2123723   total: 464ms    remaining: 496ms
483:    learn: 22.1482190   total: 465ms    remaining: 495ms
484:    learn: 22.1339636   total: 466ms    remaining: 495ms
485:    learn: 22.0631395   total: 467ms    remaining: 494ms
486:    learn: 22.0588600   total: 468ms    remaining: 493ms
487:    learn: 22.0347781   total: 469ms    remaining: 492ms
488:    learn: 22.0277106   total: 470ms    remaining: 491ms
489:    learn: 22.0139632   total: 470ms    remaining: 490ms
490:    learn: 21.9731526   total: 471ms    remaining: 489ms
491:    learn: 21.9436288   total: 472ms    remaining: 488ms
492:    learn: 21.8956250   total: 473ms    remaining: 487ms
493:    learn: 21.8193608   total: 474ms    remaining: 485ms
494:    learn: 21.8167284   total: 475ms    remaining: 485ms
495:    learn: 21.7766221   total: 476ms    remaining: 483ms
496:    learn: 21.7498528   total: 477ms    remaining: 482ms
497:    learn: 21.7303242   total: 477ms    remaining: 481ms
498:    learn: 21.6849682   total: 478ms    remaining: 480ms
499:    learn: 21.6427310   total: 479ms    remaining: 479ms
500:    learn: 21.5528544   total: 480ms    remaining: 478ms
501:    learn: 21.5054316   total: 481ms    remaining: 478ms
502:    learn: 21.4760129   total: 482ms    remaining: 477ms
503:    learn: 21.4490218   total: 483ms    remaining: 476ms
504:    learn: 21.4265026   total: 484ms    remaining: 475ms
505:    learn: 21.3902548   total: 485ms    remaining: 473ms
506:    learn: 21.3540889   total: 486ms    remaining: 472ms
507:    learn: 21.3198302   total: 487ms    remaining: 471ms
508:    learn: 21.2740171   total: 488ms    remaining: 470ms
509:    learn: 21.2345957   total: 488ms    remaining: 469ms
510:    learn: 21.1872209   total: 489ms    remaining: 468ms
511:    learn: 21.1695153   total: 490ms    remaining: 467ms
512:    learn: 21.1522472   total: 491ms    remaining: 466ms
513:    learn: 21.1497867   total: 492ms    remaining: 465ms
514:    learn: 21.1015735   total: 493ms    remaining: 464ms
515:    learn: 21.0385321   total: 494ms    remaining: 464ms
516:    learn: 20.9991473   total: 495ms    remaining: 463ms
517:    learn: 20.9703746   total: 496ms    remaining: 462ms
518:    learn: 20.9222185   total: 497ms    remaining: 461ms
519:    learn: 20.8999891   total: 498ms    remaining: 460ms
520:    learn: 20.8554804   total: 499ms    remaining: 459ms
521:    learn: 20.8128166   total: 500ms    remaining: 458ms
522:    learn: 20.7911705   total: 501ms    remaining: 457ms
523:    learn: 20.7460615   total: 501ms    remaining: 456ms
524:    learn: 20.7055146   total: 502ms    remaining: 455ms
525:    learn: 20.6579428   total: 504ms    remaining: 454ms
526:    learn: 20.6358801   total: 504ms    remaining: 453ms
527:    learn: 20.5876242   total: 505ms    remaining: 452ms
528:    learn: 20.5663373   total: 506ms    remaining: 451ms
529:    learn: 20.5187919   total: 507ms    remaining: 450ms
530:    learn: 20.4578327   total: 508ms    remaining: 449ms
531:    learn: 20.3828339   total: 509ms    remaining: 448ms
532:    learn: 20.3633039   total: 510ms    remaining: 447ms
533:    learn: 20.3358557   total: 511ms    remaining: 446ms
534:    learn: 20.3215443   total: 512ms    remaining: 445ms
535:    learn: 20.3123640   total: 513ms    remaining: 444ms
536:    learn: 20.2915068   total: 514ms    remaining: 443ms
537:    learn: 20.2365966   total: 515ms    remaining: 442ms
538:    learn: 20.2115139   total: 516ms    remaining: 441ms
539:    learn: 20.2054815   total: 517ms    remaining: 440ms
540:    learn: 20.1590883   total: 518ms    remaining: 439ms
541:    learn: 20.1547842   total: 519ms    remaining: 438ms
542:    learn: 20.1279782   total: 520ms    remaining: 437ms
543:    learn: 20.1237081   total: 521ms    remaining: 436ms
544:    learn: 20.0569792   total: 522ms    remaining: 435ms
545:    learn: 20.0329940   total: 522ms    remaining: 434ms
546:    learn: 19.9896129   total: 523ms    remaining: 433ms
547:    learn: 19.9398546   total: 524ms    remaining: 432ms
548:    learn: 19.8595835   total: 525ms    remaining: 431ms
549:    learn: 19.8409663   total: 526ms    remaining: 430ms
550:    learn: 19.8053021   total: 527ms    remaining: 430ms
551:    learn: 19.7454656   total: 529ms    remaining: 429ms
552:    learn: 19.6980410   total: 530ms    remaining: 428ms
553:    learn: 19.6626935   total: 531ms    remaining: 427ms
554:    learn: 19.6266865   total: 531ms    remaining: 426ms
555:    learn: 19.5898795   total: 533ms    remaining: 425ms
556:    learn: 19.5751403   total: 533ms    remaining: 424ms
557:    learn: 19.5731983   total: 534ms    remaining: 423ms
558:    learn: 19.5371056   total: 535ms    remaining: 422ms
559:    learn: 19.5048042   total: 536ms    remaining: 421ms
560:    learn: 19.5012571   total: 537ms    remaining: 420ms
561:    learn: 19.4832223   total: 538ms    remaining: 419ms
562:    learn: 19.4600865   total: 539ms    remaining: 418ms
563:    learn: 19.4384968   total: 540ms    remaining: 417ms
564:    learn: 19.4361093   total: 541ms    remaining: 416ms
565:    learn: 19.4110086   total: 542ms    remaining: 415ms
566:    learn: 19.3699414   total: 543ms    remaining: 414ms
567:    learn: 19.3500309   total: 544ms    remaining: 414ms
568:    learn: 19.2982027   total: 545ms    remaining: 413ms
569:    learn: 19.2285478   total: 546ms    remaining: 412ms
570:    learn: 19.2107074   total: 547ms    remaining: 411ms
571:    learn: 19.1894003   total: 548ms    remaining: 410ms
572:    learn: 19.1858983   total: 549ms    remaining: 409ms
573:    learn: 19.1385056   total: 549ms    remaining: 408ms
574:    learn: 19.1160386   total: 550ms    remaining: 407ms
575:    learn: 19.0988308   total: 551ms    remaining: 406ms
576:    learn: 19.0783785   total: 552ms    remaining: 405ms
577:    learn: 19.0231023   total: 553ms    remaining: 404ms
578:    learn: 19.0036597   total: 554ms    remaining: 403ms
579:    learn: 18.9799549   total: 555ms    remaining: 402ms
580:    learn: 18.9711749   total: 556ms    remaining: 401ms
581:    learn: 18.9443614   total: 557ms    remaining: 400ms
582:    learn: 18.9191490   total: 558ms    remaining: 399ms
583:    learn: 18.8751081   total: 559ms    remaining: 398ms
584:    learn: 18.8293391   total: 560ms    remaining: 397ms
585:    learn: 18.7965695   total: 561ms    remaining: 396ms
586:    learn: 18.7819962   total: 561ms    remaining: 395ms
587:    learn: 18.7304407   total: 562ms    remaining: 394ms
588:    learn: 18.6926858   total: 563ms    remaining: 393ms
589:    learn: 18.6510988   total: 564ms    remaining: 392ms
590:    learn: 18.6365421   total: 565ms    remaining: 391ms
591:    learn: 18.6016154   total: 566ms    remaining: 390ms
592:    learn: 18.5842759   total: 567ms    remaining: 389ms
593:    learn: 18.5401411   total: 568ms    remaining: 388ms
594:    learn: 18.5373289   total: 569ms    remaining: 387ms
595:    learn: 18.5086795   total: 570ms    remaining: 386ms
596:    learn: 18.4493193   total: 571ms    remaining: 386ms
597:    learn: 18.4063125   total: 572ms    remaining: 385ms
598:    learn: 18.3520546   total: 573ms    remaining: 384ms
599:    learn: 18.3180003   total: 574ms    remaining: 383ms
600:    learn: 18.3041324   total: 575ms    remaining: 382ms
601:    learn: 18.2722337   total: 576ms    remaining: 381ms
602:    learn: 18.2287353   total: 577ms    remaining: 380ms
603:    learn: 18.1832568   total: 578ms    remaining: 379ms
604:    learn: 18.1805032   total: 579ms    remaining: 378ms
605:    learn: 18.1520544   total: 580ms    remaining: 377ms
606:    learn: 18.1354692   total: 581ms    remaining: 376ms
607:    learn: 18.1264107   total: 581ms    remaining: 375ms
608:    learn: 18.1026188   total: 583ms    remaining: 374ms
609:    learn: 18.0549990   total: 584ms    remaining: 373ms
610:    learn: 18.0037741   total: 585ms    remaining: 372ms
611:    learn: 17.9604370   total: 586ms    remaining: 371ms
612:    learn: 17.9285276   total: 587ms    remaining: 370ms
613:    learn: 17.9213726   total: 588ms    remaining: 369ms
614:    learn: 17.9066908   total: 589ms    remaining: 368ms
615:    learn: 17.9030634   total: 590ms    remaining: 368ms
616:    learn: 17.8900287   total: 591ms    remaining: 367ms
617:    learn: 17.8829280   total: 592ms    remaining: 366ms
618:    learn: 17.8388985   total: 593ms    remaining: 365ms
619:    learn: 17.7928862   total: 593ms    remaining: 364ms
620:    learn: 17.7466362   total: 594ms    remaining: 363ms
621:    learn: 17.7104434   total: 595ms    remaining: 362ms
622:    learn: 17.6661359   total: 596ms    remaining: 361ms
623:    learn: 17.6501147   total: 597ms    remaining: 360ms
624:    learn: 17.6396963   total: 598ms    remaining: 359ms
625:    learn: 17.6102058   total: 599ms    remaining: 358ms
626:    learn: 17.6078614   total: 600ms    remaining: 357ms
627:    learn: 17.5630963   total: 601ms    remaining: 356ms
628:    learn: 17.5522732   total: 602ms    remaining: 355ms
629:    learn: 17.5429766   total: 603ms    remaining: 354ms
630:    learn: 17.5297201   total: 604ms    remaining: 353ms
631:    learn: 17.5253753   total: 605ms    remaining: 352ms
632:    learn: 17.4715324   total: 606ms    remaining: 351ms
633:    learn: 17.4328538   total: 607ms    remaining: 351ms
634:    learn: 17.4197552   total: 608ms    remaining: 350ms
635:    learn: 17.3686977   total: 609ms    remaining: 349ms
636:    learn: 17.3455865   total: 610ms    remaining: 348ms
637:    learn: 17.2964399   total: 611ms    remaining: 347ms
638:    learn: 17.2723522   total: 612ms    remaining: 346ms
639:    learn: 17.2310604   total: 613ms    remaining: 345ms
640:    learn: 17.1793967   total: 614ms    remaining: 344ms
641:    learn: 17.1565288   total: 615ms    remaining: 343ms
642:    learn: 17.1317463   total: 616ms    remaining: 342ms
643:    learn: 17.1118715   total: 617ms    remaining: 341ms
644:    learn: 17.0735889   total: 618ms    remaining: 340ms
645:    learn: 17.0529158   total: 618ms    remaining: 339ms
646:    learn: 17.0329358   total: 619ms    remaining: 338ms
647:    learn: 16.9940748   total: 620ms    remaining: 337ms
648:    learn: 16.9521366   total: 621ms    remaining: 336ms
649:    learn: 16.9500090   total: 622ms    remaining: 335ms
650:    learn: 16.9398288   total: 623ms    remaining: 334ms
651:    learn: 16.9269785   total: 624ms    remaining: 333ms
652:    learn: 16.9251987   total: 625ms    remaining: 332ms
653:    learn: 16.8865075   total: 626ms    remaining: 331ms
654:    learn: 16.8336257   total: 627ms    remaining: 330ms
655:    learn: 16.7745723   total: 628ms    remaining: 329ms
656:    learn: 16.7453018   total: 629ms    remaining: 328ms
657:    learn: 16.7132547   total: 630ms    remaining: 327ms
658:    learn: 16.6688611   total: 631ms    remaining: 326ms
659:    learn: 16.6336092   total: 631ms    remaining: 325ms
660:    learn: 16.6224205   total: 632ms    remaining: 324ms
661:    learn: 16.6047201   total: 633ms    remaining: 323ms
662:    learn: 16.5744777   total: 634ms    remaining: 322ms
663:    learn: 16.5205394   total: 635ms    remaining: 321ms
664:    learn: 16.4841640   total: 636ms    remaining: 320ms
665:    learn: 16.4718565   total: 637ms    remaining: 320ms
666:    learn: 16.4506629   total: 638ms    remaining: 319ms
667:    learn: 16.4434901   total: 639ms    remaining: 318ms
668:    learn: 16.4390434   total: 640ms    remaining: 317ms
669:    learn: 16.4033147   total: 641ms    remaining: 316ms
670:    learn: 16.3914168   total: 642ms    remaining: 315ms
671:    learn: 16.3883133   total: 643ms    remaining: 314ms
672:    learn: 16.3619882   total: 644ms    remaining: 313ms
673:    learn: 16.3187350   total: 645ms    remaining: 312ms
674:    learn: 16.3071318   total: 646ms    remaining: 311ms
675:    learn: 16.2799616   total: 646ms    remaining: 310ms
676:    learn: 16.2543052   total: 647ms    remaining: 309ms
677:    learn: 16.2247365   total: 648ms    remaining: 308ms
678:    learn: 16.2224638   total: 649ms    remaining: 307ms
679:    learn: 16.1731674   total: 651ms    remaining: 306ms
680:    learn: 16.1714538   total: 652ms    remaining: 305ms
681:    learn: 16.1613971   total: 652ms    remaining: 304ms
682:    learn: 16.1147448   total: 653ms    remaining: 303ms
683:    learn: 16.0696533   total: 654ms    remaining: 302ms
684:    learn: 16.0237966   total: 655ms    remaining: 301ms
685:    learn: 16.0150106   total: 656ms    remaining: 300ms
686:    learn: 15.9803812   total: 657ms    remaining: 299ms
687:    learn: 15.9782292   total: 658ms    remaining: 298ms
688:    learn: 15.9350017   total: 659ms    remaining: 297ms
689:    learn: 15.9165426   total: 660ms    remaining: 296ms
690:    learn: 15.9147553   total: 660ms    remaining: 295ms
691:    learn: 15.9021542   total: 661ms    remaining: 294ms
692:    learn: 15.8686086   total: 662ms    remaining: 293ms
693:    learn: 15.8668570   total: 663ms    remaining: 292ms
694:    learn: 15.8137514   total: 664ms    remaining: 291ms
695:    learn: 15.8034165   total: 665ms    remaining: 290ms
696:    learn: 15.7543075   total: 666ms    remaining: 289ms
697:    learn: 15.7227008   total: 667ms    remaining: 289ms
698:    learn: 15.6928794   total: 668ms    remaining: 288ms
699:    learn: 15.6511903   total: 669ms    remaining: 287ms
700:    learn: 15.6321406   total: 670ms    remaining: 286ms
701:    learn: 15.6043360   total: 671ms    remaining: 285ms
702:    learn: 15.5952870   total: 672ms    remaining: 284ms
703:    learn: 15.5905096   total: 673ms    remaining: 283ms
704:    learn: 15.5836839   total: 673ms    remaining: 282ms
705:    learn: 15.5652754   total: 674ms    remaining: 281ms
706:    learn: 15.5282494   total: 675ms    remaining: 280ms
707:    learn: 15.4869307   total: 676ms    remaining: 279ms
708:    learn: 15.4644509   total: 677ms    remaining: 278ms
709:    learn: 15.4380354   total: 678ms    remaining: 277ms
710:    learn: 15.4270000   total: 679ms    remaining: 276ms
711:    learn: 15.4189790   total: 680ms    remaining: 275ms
712:    learn: 15.4178505   total: 680ms    remaining: 274ms
713:    learn: 15.3923931   total: 681ms    remaining: 273ms
714:    learn: 15.3478757   total: 682ms    remaining: 272ms
715:    learn: 15.3082102   total: 683ms    remaining: 271ms
716:    learn: 15.2968684   total: 685ms    remaining: 270ms
717:    learn: 15.2819067   total: 686ms    remaining: 269ms
718:    learn: 15.2708815   total: 687ms    remaining: 268ms
719:    learn: 15.2426082   total: 688ms    remaining: 267ms
720:    learn: 15.2010081   total: 689ms    remaining: 266ms
721:    learn: 15.1757935   total: 690ms    remaining: 266ms
722:    learn: 15.1496573   total: 691ms    remaining: 265ms
723:    learn: 15.1069891   total: 691ms    remaining: 264ms
724:    learn: 15.0957545   total: 692ms    remaining: 263ms
725:    learn: 15.0678419   total: 693ms    remaining: 262ms
726:    learn: 15.0323543   total: 694ms    remaining: 261ms
727:    learn: 15.0099373   total: 695ms    remaining: 260ms
728:    learn: 14.9804890   total: 696ms    remaining: 259ms
729:    learn: 14.9526908   total: 698ms    remaining: 258ms
730:    learn: 14.9302848   total: 699ms    remaining: 257ms
731:    learn: 14.8966998   total: 700ms    remaining: 256ms
732:    learn: 14.8699435   total: 701ms    remaining: 255ms
733:    learn: 14.8304247   total: 702ms    remaining: 254ms
734:    learn: 14.7929834   total: 703ms    remaining: 253ms
735:    learn: 14.7696209   total: 704ms    remaining: 253ms
736:    learn: 14.7562766   total: 705ms    remaining: 252ms
737:    learn: 14.7213105   total: 706ms    remaining: 251ms
738:    learn: 14.7000863   total: 707ms    remaining: 250ms
739:    learn: 14.6493550   total: 708ms    remaining: 249ms
740:    learn: 14.6335162   total: 709ms    remaining: 248ms
741:    learn: 14.5998758   total: 710ms    remaining: 247ms
742:    learn: 14.5869238   total: 711ms    remaining: 246ms
743:    learn: 14.5759591   total: 712ms    remaining: 245ms
744:    learn: 14.5547290   total: 714ms    remaining: 244ms
745:    learn: 14.5283070   total: 715ms    remaining: 243ms
746:    learn: 14.4853954   total: 716ms    remaining: 242ms
747:    learn: 14.4757814   total: 717ms    remaining: 241ms
748:    learn: 14.4354938   total: 718ms    remaining: 241ms
749:    learn: 14.4036861   total: 719ms    remaining: 240ms
750:    learn: 14.3687833   total: 720ms    remaining: 239ms
751:    learn: 14.3437669   total: 721ms    remaining: 238ms
752:    learn: 14.3209004   total: 722ms    remaining: 237ms
753:    learn: 14.2981607   total: 723ms    remaining: 236ms
754:    learn: 14.2959300   total: 724ms    remaining: 235ms
755:    learn: 14.2680160   total: 724ms    remaining: 234ms
756:    learn: 14.2535389   total: 725ms    remaining: 233ms
757:    learn: 14.2501520   total: 727ms    remaining: 232ms
758:    learn: 14.2260538   total: 728ms    remaining: 231ms
759:    learn: 14.2234442   total: 729ms    remaining: 230ms
760:    learn: 14.1873135   total: 730ms    remaining: 229ms
761:    learn: 14.1765115   total: 731ms    remaining: 228ms
762:    learn: 14.1697434   total: 731ms    remaining: 227ms
763:    learn: 14.1365525   total: 732ms    remaining: 226ms
764:    learn: 14.1015278   total: 733ms    remaining: 225ms
765:    learn: 14.0622675   total: 734ms    remaining: 224ms
766:    learn: 14.0409553   total: 735ms    remaining: 223ms
767:    learn: 14.0337570   total: 736ms    remaining: 222ms
768:    learn: 13.9910329   total: 737ms    remaining: 221ms
769:    learn: 13.9510113   total: 738ms    remaining: 220ms
770:    learn: 13.9174679   total: 739ms    remaining: 220ms
771:    learn: 13.8864002   total: 740ms    remaining: 219ms
772:    learn: 13.8446475   total: 741ms    remaining: 218ms
773:    learn: 13.8397236   total: 742ms    remaining: 217ms
774:    learn: 13.8029820   total: 743ms    remaining: 216ms
775:    learn: 13.7857820   total: 744ms    remaining: 215ms
776:    learn: 13.7646024   total: 745ms    remaining: 214ms
777:    learn: 13.7486851   total: 746ms    remaining: 213ms
778:    learn: 13.7014327   total: 747ms    remaining: 212ms
779:    learn: 13.6934358   total: 747ms    remaining: 211ms
780:    learn: 13.6683386   total: 748ms    remaining: 210ms
781:    learn: 13.6566338   total: 749ms    remaining: 209ms
782:    learn: 13.6444486   total: 750ms    remaining: 208ms
783:    learn: 13.6250279   total: 751ms    remaining: 207ms
784:    learn: 13.5875176   total: 752ms    remaining: 206ms
785:    learn: 13.5783446   total: 752ms    remaining: 205ms
786:    learn: 13.5576008   total: 753ms    remaining: 204ms
787:    learn: 13.5477919   total: 754ms    remaining: 203ms
788:    learn: 13.5055189   total: 755ms    remaining: 202ms
789:    learn: 13.4644475   total: 756ms    remaining: 201ms
790:    learn: 13.4570524   total: 757ms    remaining: 200ms
791:    learn: 13.4244055   total: 758ms    remaining: 199ms
792:    learn: 13.4171601   total: 759ms    remaining: 198ms
793:    learn: 13.3977292   total: 760ms    remaining: 197ms
794:    learn: 13.3911146   total: 761ms    remaining: 196ms
795:    learn: 13.3816584   total: 762ms    remaining: 195ms
796:    learn: 13.3539009   total: 763ms    remaining: 194ms
797:    learn: 13.3166557   total: 764ms    remaining: 193ms
798:    learn: 13.2942330   total: 765ms    remaining: 192ms
799:    learn: 13.2692862   total: 765ms    remaining: 191ms
800:    learn: 13.2547553   total: 766ms    remaining: 190ms
801:    learn: 13.2259343   total: 767ms    remaining: 189ms
802:    learn: 13.2246521   total: 768ms    remaining: 188ms
803:    learn: 13.2119291   total: 769ms    remaining: 188ms
804:    learn: 13.1793474   total: 771ms    remaining: 187ms
805:    learn: 13.1652542   total: 772ms    remaining: 186ms
806:    learn: 13.1293534   total: 773ms    remaining: 185ms
807:    learn: 13.1211653   total: 774ms    remaining: 184ms
808:    learn: 13.1110732   total: 775ms    remaining: 183ms
809:    learn: 13.1056131   total: 776ms    remaining: 182ms
810:    learn: 13.0704954   total: 776ms    remaining: 181ms
811:    learn: 13.0482918   total: 777ms    remaining: 180ms
812:    learn: 13.0313717   total: 778ms    remaining: 179ms
813:    learn: 13.0295572   total: 779ms    remaining: 178ms
814:    learn: 13.0076548   total: 780ms    remaining: 177ms
815:    learn: 13.0056559   total: 781ms    remaining: 176ms
816:    learn: 12.9992709   total: 782ms    remaining: 175ms
817:    learn: 12.9824007   total: 783ms    remaining: 174ms
818:    learn: 12.9670734   total: 784ms    remaining: 173ms
819:    learn: 12.9372833   total: 785ms    remaining: 172ms
820:    learn: 12.9087596   total: 786ms    remaining: 171ms
821:    learn: 12.9075563   total: 787ms    remaining: 170ms
822:    learn: 12.8694642   total: 788ms    remaining: 169ms
823:    learn: 12.8683302   total: 789ms    remaining: 168ms
824:    learn: 12.8490869   total: 790ms    remaining: 168ms
825:    learn: 12.8290022   total: 791ms    remaining: 167ms
826:    learn: 12.8107471   total: 792ms    remaining: 166ms
827:    learn: 12.7834375   total: 793ms    remaining: 165ms
828:    learn: 12.7492612   total: 794ms    remaining: 164ms
829:    learn: 12.7143908   total: 795ms    remaining: 163ms
830:    learn: 12.6794802   total: 796ms    remaining: 162ms
831:    learn: 12.6575840   total: 796ms    remaining: 161ms
832:    learn: 12.6414062   total: 797ms    remaining: 160ms
833:    learn: 12.6323550   total: 798ms    remaining: 159ms
834:    learn: 12.6004844   total: 799ms    remaining: 158ms
835:    learn: 12.5992796   total: 800ms    remaining: 157ms
836:    learn: 12.5664653   total: 801ms    remaining: 156ms
837:    learn: 12.5389714   total: 802ms    remaining: 155ms
838:    learn: 12.5116178   total: 803ms    remaining: 154ms
839:    learn: 12.4901320   total: 803ms    remaining: 153ms
840:    learn: 12.4812527   total: 805ms    remaining: 152ms
841:    learn: 12.4562707   total: 805ms    remaining: 151ms
842:    learn: 12.4209878   total: 806ms    remaining: 150ms
843:    learn: 12.4015992   total: 807ms    remaining: 149ms
844:    learn: 12.3675998   total: 808ms    remaining: 148ms
845:    learn: 12.3521691   total: 809ms    remaining: 147ms
846:    learn: 12.3426886   total: 810ms    remaining: 146ms
847:    learn: 12.3188571   total: 811ms    remaining: 145ms
848:    learn: 12.2937388   total: 812ms    remaining: 144ms
849:    learn: 12.2724402   total: 813ms    remaining: 143ms
850:    learn: 12.2342559   total: 814ms    remaining: 142ms
851:    learn: 12.2159725   total: 815ms    remaining: 142ms
852:    learn: 12.2147289   total: 816ms    remaining: 141ms
853:    learn: 12.1935801   total: 817ms    remaining: 140ms
854:    learn: 12.1795597   total: 818ms    remaining: 139ms
855:    learn: 12.1489328   total: 819ms    remaining: 138ms
856:    learn: 12.1476791   total: 820ms    remaining: 137ms
857:    learn: 12.1143878   total: 821ms    remaining: 136ms
858:    learn: 12.1133877   total: 821ms    remaining: 135ms
859:    learn: 12.0927666   total: 822ms    remaining: 134ms
860:    learn: 12.0908934   total: 823ms    remaining: 133ms
861:    learn: 12.0667739   total: 824ms    remaining: 132ms
862:    learn: 12.0312513   total: 825ms    remaining: 131ms
863:    learn: 12.0298478   total: 826ms    remaining: 130ms
864:    learn: 12.0174484   total: 827ms    remaining: 129ms
865:    learn: 12.0093901   total: 828ms    remaining: 128ms
866:    learn: 11.9731302   total: 829ms    remaining: 127ms
867:    learn: 11.9617694   total: 830ms    remaining: 126ms
868:    learn: 11.9536413   total: 831ms    remaining: 125ms
869:    learn: 11.9234807   total: 832ms    remaining: 124ms
870:    learn: 11.8995296   total: 833ms    remaining: 123ms
871:    learn: 11.8985399   total: 833ms    remaining: 122ms
872:    learn: 11.8748993   total: 834ms    remaining: 121ms
873:    learn: 11.8473937   total: 835ms    remaining: 120ms
874:    learn: 11.7980081   total: 836ms    remaining: 119ms
875:    learn: 11.7968597   total: 837ms    remaining: 119ms
876:    learn: 11.7952536   total: 838ms    remaining: 118ms
877:    learn: 11.7765712   total: 839ms    remaining: 117ms
878:    learn: 11.7433623   total: 840ms    remaining: 116ms
879:    learn: 11.7220227   total: 841ms    remaining: 115ms
880:    learn: 11.7079171   total: 842ms    remaining: 114ms
881:    learn: 11.6892547   total: 843ms    remaining: 113ms
882:    learn: 11.6752283   total: 844ms    remaining: 112ms
883:    learn: 11.6465248   total: 845ms    remaining: 111ms
884:    learn: 11.6199604   total: 846ms    remaining: 110ms
885:    learn: 11.5840290   total: 847ms    remaining: 109ms
886:    learn: 11.5665343   total: 848ms    remaining: 108ms
887:    learn: 11.5653334   total: 849ms    remaining: 107ms
888:    learn: 11.5409858   total: 850ms    remaining: 106ms
889:    learn: 11.5126963   total: 850ms    remaining: 105ms
890:    learn: 11.4937102   total: 851ms    remaining: 104ms
891:    learn: 11.4652907   total: 852ms    remaining: 103ms
892:    learn: 11.4406454   total: 853ms    remaining: 102ms
893:    learn: 11.4157600   total: 854ms    remaining: 101ms
894:    learn: 11.4145225   total: 855ms    remaining: 100ms
895:    learn: 11.3944764   total: 856ms    remaining: 99.4ms
896:    learn: 11.3718753   total: 857ms    remaining: 98.4ms
897:    learn: 11.3698629   total: 858ms    remaining: 97.5ms
898:    learn: 11.3652583   total: 859ms    remaining: 96.5ms
899:    learn: 11.3403921   total: 860ms    remaining: 95.6ms
900:    learn: 11.3188723   total: 861ms    remaining: 94.6ms
901:    learn: 11.2825028   total: 862ms    remaining: 93.6ms
902:    learn: 11.2756316   total: 863ms    remaining: 92.7ms
903:    learn: 11.2746247   total: 864ms    remaining: 91.7ms
904:    learn: 11.2506409   total: 864ms    remaining: 90.7ms
905:    learn: 11.2459222   total: 865ms    remaining: 89.8ms
906:    learn: 11.2231937   total: 866ms    remaining: 88.8ms
907:    learn: 11.2191230   total: 867ms    remaining: 87.8ms
908:    learn: 11.1878231   total: 868ms    remaining: 86.9ms
909:    learn: 11.1635473   total: 869ms    remaining: 86ms
910:    learn: 11.1593446   total: 870ms    remaining: 85ms
911:    learn: 11.1312444   total: 871ms    remaining: 84.1ms
912:    learn: 11.1178570   total: 872ms    remaining: 83.1ms
913:    learn: 11.1042827   total: 873ms    remaining: 82.1ms
914:    learn: 11.0835715   total: 874ms    remaining: 81.2ms
915:    learn: 11.0643196   total: 875ms    remaining: 80.2ms
916:    learn: 11.0602889   total: 876ms    remaining: 79.3ms
917:    learn: 11.0272219   total: 877ms    remaining: 78.3ms
918:    learn: 10.9986897   total: 878ms    remaining: 77.3ms
919:    learn: 10.9792479   total: 879ms    remaining: 76.4ms
920:    learn: 10.9535501   total: 879ms    remaining: 75.4ms
921:    learn: 10.9275044   total: 880ms    remaining: 74.5ms
922:    learn: 10.9219224   total: 881ms    remaining: 73.5ms
923:    learn: 10.9035961   total: 882ms    remaining: 72.6ms
924:    learn: 10.8891537   total: 884ms    remaining: 71.6ms
925:    learn: 10.8702443   total: 885ms    remaining: 70.7ms
926:    learn: 10.8421716   total: 886ms    remaining: 69.7ms
927:    learn: 10.8203110   total: 886ms    remaining: 68.8ms
928:    learn: 10.7915801   total: 888ms    remaining: 67.8ms
929:    learn: 10.7652304   total: 888ms    remaining: 66.9ms
930:    learn: 10.7342804   total: 889ms    remaining: 65.9ms
931:    learn: 10.7211866   total: 890ms    remaining: 65ms
932:    learn: 10.7047031   total: 891ms    remaining: 64ms
933:    learn: 10.6961734   total: 892ms    remaining: 63ms
934:    learn: 10.6841913   total: 893ms    remaining: 62.1ms
935:    learn: 10.6763968   total: 894ms    remaining: 61.1ms
936:    learn: 10.6645409   total: 895ms    remaining: 60.2ms
937:    learn: 10.6631696   total: 896ms    remaining: 59.2ms
938:    learn: 10.6586104   total: 896ms    remaining: 58.2ms
939:    learn: 10.6395476   total: 897ms    remaining: 57.3ms
940:    learn: 10.6354434   total: 898ms    remaining: 56.3ms
941:    learn: 10.6184324   total: 899ms    remaining: 55.4ms
942:    learn: 10.6055869   total: 900ms    remaining: 54.4ms
943:    learn: 10.5965561   total: 901ms    remaining: 53.5ms
944:    learn: 10.5955793   total: 902ms    remaining: 52.5ms
945:    learn: 10.5947611   total: 903ms    remaining: 51.6ms
946:    learn: 10.5693144   total: 904ms    remaining: 50.6ms
947:    learn: 10.5637694   total: 905ms    remaining: 49.6ms
948:    learn: 10.5374266   total: 906ms    remaining: 48.7ms
949:    learn: 10.5099668   total: 907ms    remaining: 47.7ms
950:    learn: 10.5086617   total: 908ms    remaining: 46.8ms
951:    learn: 10.5067682   total: 909ms    remaining: 45.8ms
952:    learn: 10.4925699   total: 909ms    remaining: 44.9ms
953:    learn: 10.4716689   total: 911ms    remaining: 43.9ms
954:    learn: 10.4379081   total: 912ms    remaining: 43ms
955:    learn: 10.4006751   total: 913ms    remaining: 42ms
956:    learn: 10.3836660   total: 913ms    remaining: 41ms
957:    learn: 10.3575722   total: 915ms    remaining: 40.1ms
958:    learn: 10.3478138   total: 916ms    remaining: 39.1ms
959:    learn: 10.3187226   total: 916ms    remaining: 38.2ms
960:    learn: 10.3022324   total: 918ms    remaining: 37.2ms
961:    learn: 10.2997720   total: 919ms    remaining: 36.3ms
962:    learn: 10.2989286   total: 920ms    remaining: 35.3ms
963:    learn: 10.2895343   total: 921ms    remaining: 34.4ms
964:    learn: 10.2815641   total: 922ms    remaining: 33.4ms
965:    learn: 10.2625952   total: 923ms    remaining: 32.5ms
966:    learn: 10.2506695   total: 924ms    remaining: 31.5ms
967:    learn: 10.2397609   total: 925ms    remaining: 30.6ms
968:    learn: 10.2154145   total: 927ms    remaining: 29.6ms
969:    learn: 10.2143644   total: 928ms    remaining: 28.7ms
970:    learn: 10.1981073   total: 929ms    remaining: 27.7ms
971:    learn: 10.1920221   total: 930ms    remaining: 26.8ms
972:    learn: 10.1806991   total: 931ms    remaining: 25.8ms
973:    learn: 10.1597212   total: 933ms    remaining: 24.9ms
974:    learn: 10.1358828   total: 934ms    remaining: 23.9ms
975:    learn: 10.1222784   total: 935ms    remaining: 23ms
976:    learn: 10.1128777   total: 936ms    remaining: 22ms
977:    learn: 10.1065225   total: 937ms    remaining: 21.1ms
978:    learn: 10.0954545   total: 938ms    remaining: 20.1ms
979:    learn: 10.0725280   total: 939ms    remaining: 19.2ms
980:    learn: 10.0443783   total: 940ms    remaining: 18.2ms
981:    learn: 10.0421942   total: 941ms    remaining: 17.2ms
982:    learn: 10.0135134   total: 941ms    remaining: 16.3ms
983:    learn: 10.0009906   total: 942ms    remaining: 15.3ms
984:    learn: 9.9894585    total: 943ms    remaining: 14.4ms
985:    learn: 9.9872832    total: 944ms    remaining: 13.4ms
986:    learn: 9.9654572    total: 945ms    remaining: 12.4ms
987:    learn: 9.9461931    total: 946ms    remaining: 11.5ms
988:    learn: 9.9451290    total: 947ms    remaining: 10.5ms
989:    learn: 9.9431210    total: 948ms    remaining: 9.57ms
990:    learn: 9.9299945    total: 949ms    remaining: 8.62ms
991:    learn: 9.9131150    total: 950ms    remaining: 7.66ms
992:    learn: 9.9123473    total: 950ms    remaining: 6.7ms
993:    learn: 9.8978089    total: 951ms    remaining: 5.74ms
994:    learn: 9.8841806    total: 952ms    remaining: 4.79ms
995:    learn: 9.8638563    total: 953ms    remaining: 3.83ms
996:    learn: 9.8491248    total: 954ms    remaining: 2.87ms
997:    learn: 9.8427010    total: 955ms    remaining: 1.91ms
998:    learn: 9.8417925    total: 956ms    remaining: 957us
999:    learn: 9.8380086    total: 957ms    remaining: 0us
Mean absolute error: 43.06
Mean absolute percentage error: 39.39%
Root Mean squared error: 55.14
Coefficient of determination: 0.45

plot_residuals(diabetes_y_test, diabetes_y_pred)

10 Compare Results

results_df = pd.DataFrame(results_list)

results_df.sort_values('MAPE')

	model	RMSE	MAPE
6	CatBoost	55.137472	0.393867
4	XGBoost	58.011463	0.398371
3	Random Forest	54.959277	0.403435
5	Light GBM	57.971124	0.407147
1	Multiple Linear Regression	54.253023	0.407386
0	Linear Regression	62.004712	0.488760
2	Decision Tree	76.000948	0.506753

Let’s remind ourselves - what’s the scale of the output feature?

diabetes_y_test.hist()

diabetes_y_train.hist()

10.1 Plotting Residuals

Plotting the residuals (errors) can help us to understand whether the model is consistently over or under estimating.