我们用tensorflow2来看一个服饰数据集的识别。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    print(x_valid.shape, y_valid.shape)
    print(x_train.shape, y_train.shape)
    print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    show_single_image(x_train[1])

运行结果

(5000, 28, 28) (5000,)
(55000, 28, 28) (55000,)
(10000, 28, 28) (10000,)

这里我们可以看到，这些图像数据集都是28*28的单通道图片。我们将这些数据集分成了55000个训练数据集，5000个验证数据集以及10000个测试数据集。现在我们再来看一下这些数据集的分类类型

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    show_imgs(3, 5, x_train, y_train, class_names)

运行结果

这里我们可以看到这些服饰总共有10种类型。现在我们就使用神经网络来对这些数据进行训练和验证

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵损失函数，随机梯度下降法进行训练
    # 这里需要说明的是如果是categorical_crossentropy
    # 则需要将标签数据集进行one_hot处理，如果是sparse_categorical_crossentropy，
    # 则不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    history = model.fit(x_train, y_train, epochs=10, validation_data=(x_valid, y_valid))
    print(model.summary())
    print(history.history)

    def plot_learning_curves(history):
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
55000/55000 [==============================] - 6s 107us/sample - loss: 2.2443 - accuracy: 0.7192 - val_loss: 0.6124 - val_accuracy: 0.7928
Epoch 2/10
55000/55000 [==============================] - 5s 95us/sample - loss: 0.5678 - accuracy: 0.7980 - val_loss: 0.5467 - val_accuracy: 0.8018
Epoch 3/10
55000/55000 [==============================] - 5s 92us/sample - loss: 0.4991 - accuracy: 0.8188 - val_loss: 0.4968 - val_accuracy: 0.8358
Epoch 4/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.4646 - accuracy: 0.8316 - val_loss: 0.4910 - val_accuracy: 0.8332
Epoch 5/10
55000/55000 [==============================] - 7s 119us/sample - loss: 0.4371 - accuracy: 0.8393 - val_loss: 0.4768 - val_accuracy: 0.8374
Epoch 6/10
55000/55000 [==============================] - 7s 123us/sample - loss: 0.4211 - accuracy: 0.8455 - val_loss: 0.4608 - val_accuracy: 0.8444
Epoch 7/10
55000/55000 [==============================] - 6s 103us/sample - loss: 0.4046 - accuracy: 0.8504 - val_loss: 0.4506 - val_accuracy: 0.8464
Epoch 8/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.3902 - accuracy: 0.8550 - val_loss: 0.4427 - val_accuracy: 0.8480
Epoch 9/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.3796 - accuracy: 0.8581 - val_loss: 0.4814 - val_accuracy: 0.8312
Epoch 10/10
55000/55000 [==============================] - 5s 99us/sample - loss: 0.3714 - accuracy: 0.8608 - val_loss: 0.4665 - val_accuracy: 0.8440
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 266,610
Trainable params: 266,610
Non-trainable params: 0
_________________________________________________________________
None
{'loss': [2.2442730520161716, 0.5677844335772775, 0.49907195046598263, 0.46455069565122775, 0.43706936968456617, 0.4210625493981621, 0.4045901944290508, 0.3901773897236044, 0.3795636681730097, 0.37135513420971955], 'accuracy': [0.71916366, 0.7979818, 0.8187636, 0.83156365, 0.8393091, 0.84549093, 0.8503636, 0.8550182, 0.8580545, 0.8608364], 'val_loss': [0.6123501856803895, 0.5466709545612335, 0.4968059474229813, 0.49100303683280944, 0.4768493502855301, 0.46080493590831756, 0.4505628205776215, 0.44266556935310364, 0.4814212405562401, 0.4664571787118912], 'val_accuracy': [0.7928, 0.8018, 0.8358, 0.8332, 0.8374, 0.8444, 0.8464, 0.848, 0.8312, 0.844]}

这里我们使用了三个全连接层来构建神经网络，对验证数据集的准确率达到了85.92%。这里我们使用的是随机梯度下降法，关于随机梯度下降法的内容请参考机器学习算法整理 中的随机梯度下降法。下面这个报告是告诉我们在神经网络中每一个全连接层的维度以及参数量。由于我们的图像是28*28，所以扁平化处理后是28*28=784个维度。经过全连接层之后，我们要把第一个全连接层的输出变成样本数*300个维度的矩阵，我们需要将矩阵[样本数，784]•[784, 300]的矩阵W(有关矩阵乘法的内容请参考线性代数整理 中矩阵和矩阵的乘法)，一般我们会加上一个b，就是一个长度为300的向量。这样就得到了一个[样本数, 300]的输出矩阵。而参数量就等于784*300+300=235500。同理，第二个全连接层的输出为样本数*100的矩阵，我们需要将第一层的输出矩阵[样本数, 300]•[300, 100]+b，就得到[样本数, 100]的输出，参数量即为300*100+100=30100。而图中也显示出损失函数值是在稳固下降的，而识别准确率是在稳固上升的。现在我们来给训练数据加上归一化处理。我们先来看一下归一化前后训练数据集的最大值和最小值

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 归一化过程中，需要将三维图片数据转成二维数据[None, 28, 28] -> [None, 784]
    # 归一化结束之后再转回三维图片数据,这里fit_transform会记录训练数据集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 这里验证数据集会采用训练数据集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    print(np.max(x_train), np.min(x_train))

运行结果

255 0
2.0231433 -0.8105136

这里我们可以看到，训练数据集归一化前后它们的最大值最小值是不同的。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 归一化过程中，需要将三维图片数据转成二维数据[None, 28, 28] -> [None, 784]
    # 归一化结束之后再转回三维图片数据,这里fit_transform会记录训练数据集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 这里验证数据集会采用训练数据集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵损失函数，随机梯度下降法进行训练
    # 这里需要说明的是如果是categorical_crossentropy
    # 则需要将标签数据集进行one_hot处理，如果是sparse_categorical_crossentropy，
    # 则不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    history = model.fit(x_train, y_train, epochs=10, validation_data=(x_valid, y_valid))
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    print(history.history)

    def plot_learning_curves(history):
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
55000/55000 [==============================] - 5s 95us/sample - loss: 0.9122 - accuracy: 0.7003 - val_loss: 0.6157 - val_accuracy: 0.7920
Epoch 2/10
55000/55000 [==============================] - 5s 83us/sample - loss: 0.5771 - accuracy: 0.8010 - val_loss: 0.5165 - val_accuracy: 0.8182
Epoch 3/10
55000/55000 [==============================] - 6s 104us/sample - loss: 0.5075 - accuracy: 0.8235 - val_loss: 0.4732 - val_accuracy: 0.8378
Epoch 4/10
55000/55000 [==============================] - 7s 127us/sample - loss: 0.4716 - accuracy: 0.8343 - val_loss: 0.4473 - val_accuracy: 0.8480
Epoch 5/10
55000/55000 [==============================] - 6s 117us/sample - loss: 0.4477 - accuracy: 0.8421 - val_loss: 0.4323 - val_accuracy: 0.8530
Epoch 6/10
55000/55000 [==============================] - 6s 110us/sample - loss: 0.4306 - accuracy: 0.8483 - val_loss: 0.4175 - val_accuracy: 0.8580
Epoch 7/10
55000/55000 [==============================] - 5s 90us/sample - loss: 0.4167 - accuracy: 0.8521 - val_loss: 0.4101 - val_accuracy: 0.8564
Epoch 8/10
55000/55000 [==============================] - 5s 90us/sample - loss: 0.4059 - accuracy: 0.8562 - val_loss: 0.4022 - val_accuracy: 0.8626
Epoch 9/10
55000/55000 [==============================] - 5s 92us/sample - loss: 0.3964 - accuracy: 0.8599 - val_loss: 0.3944 - val_accuracy: 0.8634
Epoch 10/10
55000/55000 [==============================] - 5s 93us/sample - loss: 0.3880 - accuracy: 0.8630 - val_loss: 0.3889 - val_accuracy: 0.8606
10000/1 ==============================================] - 1s 51us/sample - loss: 0.2991 - accuracy: 0.8472
[0.43079993045330045, 0.8472]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 266,610
Trainable params: 266,610
Non-trainable params: 0
_________________________________________________________________
None
{'loss': [0.912204550890489, 0.5771390253067017, 0.507524390927228, 0.4716112705404108, 0.4477458443338221, 0.4306134765581651, 0.416733508162065, 0.4059152506828308, 0.3963754140810533, 0.38801397878473454], 'accuracy': [0.70034546, 0.801, 0.82345456, 0.8343091, 0.84214544, 0.84827274, 0.85214543, 0.85623634, 0.8598727, 0.8629818], 'val_loss': [0.615698953294754, 0.516535887670517, 0.47322214045524597, 0.4472690547943115, 0.43233813087940215, 0.41751169788837433, 0.41005783474445345, 0.40220267939567567, 0.3943846334218979, 0.38885269825458524], 'val_accuracy': [0.792, 0.8182, 0.8378, 0.848, 0.853, 0.858, 0.8564, 0.8626, 0.8634, 0.8606]}

这里我们对测试数据集进行了测试，它的分类准确率达到了84.72%。

回调函数

回调函数是当我们在训练模型的时候，中间可能要做一些事情。比如说模型训练中，当损失函数值loss不再下降的时候，我们可以提前停止训练。又比如在训练的过程中，每隔段时间把模型参数给保存下来。又或者在模型训练过程中，就对一些模型指标进行图形化输出等等。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 归一化过程中，需要将三维图片数据转成二维数据[None, 28, 28] -> [None, 784]
    # 归一化结束之后再转回三维图片数据,这里fit_transform会记录训练数据集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 这里验证数据集会采用训练数据集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵损失函数，随机梯度下降法进行训练
    # 这里需要说明的是如果是categorical_crossentropy
    # 则需要将标签数据集进行one_hot处理，如果是sparse_categorical_crossentropy，
    # 则不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我们这里使用三个回调函数Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以保存我们在训练中的指标，比如损失函数值以及识别准确率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是保存模型参数
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前终止训练
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=10,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

当我们训练完成之后，可以到callbacks上一层文件夹中，展开callbacks文件夹，我们可以看到这样一些文件

在callbacks上一层文件夹中输入命令

(base) -bash-3.2$ tensorboard --logdir=callbacks

会得到这样一些输出

Serving TensorBoard on localhost; to expose to the network, use a proxy or pass --bind_all
TensorBoard 2.0.0 at http://localhost:6006/ (Press CTRL+C to quit)

此时打开浏览器访问http://localhost:6006，会看到我们训练中的指标数据图

以及神经网络的层级图

而EarlyStopping的参数意义如下：

1. min_delta：增大或减小的阈值，只有大于这个部分才算作improvement(改善)。这个值的大小取决于monitor，也反映了你的容忍程度。例如monitor(监控)是’acc’，同时其变化范围在70%-90%之间，所以对于小于0.01%的变化不关心。加上观察到训练过程中存在抖动的情况（即先下降后上升），所以适当增大容忍程度，最终设为0.001%。
2. patience：能够容忍多少个epoch内都没有improvement。这个设置其实是在抖动和真正的准确率下降之间做tradeoff(权衡)。如果patience设的大，那么最终得到的准确率要略低于模型可以达到的最高准确率。如果patience设的小，那么模型很可能在前期抖动，还在全图搜索的阶段就停止了，准确率一般很差。patience的大小和learning rate直接相关。在learning rate设定的情况下，前期先训练几次观察抖动的epoch number，比其稍大些设置patience。在learning rate变化的情况下，建议要略小于最大的抖动epoch number。笔者在引入EarlyStopping之前就已经得到可以接受的结果了，EarlyStopping算是锦上添花，所以patience设的比较高，设为抖动epoch number的最大值。
3. min_delta和patience都和“避免模型停止在抖动过程中”有关系，所以调节的时候需要互相协调。通常情况下，min_delta降低，那么patience可以适当减少；min_delta增加，那么patience需要适当延长；反之亦然。

归一化与批归一化

• 归一化

1. Min-max归一化：x*=(x-min)/(max-min)
2. Z-score归一化：x*=(x-μ)/δ，这里μ表示均值，δ表示方差，是我们经常使用的归一化

• 批归一化

批归一化是指把归一化这种思路从输入数据上给扩展到网络的每层的激活值上，每层的激活值都做归一化。我们知道对于神经网络来说是一种层级结构，每一层的输出就是下一层的输入，我们在每一层的输入上都去做归一化，那么就会使得整个的网络的效果会更好。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 归一化过程中，需要将三维图片数据转成二维数据[None, 28, 28] -> [None, 784]
    # 归一化结束之后再转回三维图片数据,这里fit_transform会记录训练数据集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 这里验证数据集会采用训练数据集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        # 批归一化
        keras.layers.BatchNormalization(),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.BatchNormalization(),
        keras.layers.Dense(10, activation='softmax')
        # 上面是将批归一化放在激活函数之后，也可以放在激活函数之前
        # keras.layers.Dense(300),
        # keras.layers.BatchNormalization(),
        # keras.layers.Activation('relu'),
    ])
    # 使用交叉熵损失函数，随机梯度下降法进行训练
    # 这里需要说明的是如果是categorical_crossentropy
    # 则需要将标签数据集进行one_hot处理，如果是sparse_categorical_crossentropy，
    # 则不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我们这里使用三个回调函数Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以保存我们在训练中的指标，比如损失函数值以及识别准确率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是保存模型参数
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前终止训练
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=10,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

运行结果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
   32/55000 [..............................] - ETA: 44:06 - loss: 3.0657 - accuracy: 0.06252021-10-20 23:21:46.915226: I tensorflow/core/profiler/lib/profiler_session.cc:184] Profiler session started.
55000/55000 [==============================] - 12s 223us/sample - loss: 0.8149 - accuracy: 0.7291 - val_loss: 0.5425 - val_accuracy: 0.8172
Epoch 2/10
55000/55000 [==============================] - 11s 195us/sample - loss: 0.5600 - accuracy: 0.8085 - val_loss: 0.4776 - val_accuracy: 0.8398
Epoch 3/10
55000/55000 [==============================] - 13s 229us/sample - loss: 0.5023 - accuracy: 0.8254 - val_loss: 0.4405 - val_accuracy: 0.8514
Epoch 4/10
55000/55000 [==============================] - 12s 216us/sample - loss: 0.4684 - accuracy: 0.8367 - val_loss: 0.4181 - val_accuracy: 0.8600
Epoch 5/10
55000/55000 [==============================] - 11s 200us/sample - loss: 0.4447 - accuracy: 0.8459 - val_loss: 0.4045 - val_accuracy: 0.8634
Epoch 6/10
55000/55000 [==============================] - 11s 200us/sample - loss: 0.4250 - accuracy: 0.8526 - val_loss: 0.3916 - val_accuracy: 0.8688
Epoch 7/10
55000/55000 [==============================] - 11s 204us/sample - loss: 0.4093 - accuracy: 0.8561 - val_loss: 0.3838 - val_accuracy: 0.8692
Epoch 8/10
55000/55000 [==============================] - 11s 207us/sample - loss: 0.3983 - accuracy: 0.8600 - val_loss: 0.3772 - val_accuracy: 0.8718
Epoch 9/10
55000/55000 [==============================] - 11s 207us/sample - loss: 0.3886 - accuracy: 0.8645 - val_loss: 0.3687 - val_accuracy: 0.8732
Epoch 10/10
55000/55000 [==============================] - 12s 212us/sample - loss: 0.3777 - accuracy: 0.8681 - val_loss: 0.3646 - val_accuracy: 0.8742
10000/1 [==================================================] - 1s 99us/sample - loss: 0.3293 - accuracy: 0.8609
[0.39349488220214845, 0.8609]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
batch_normalization (BatchNo (None, 300)               1200      
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
batch_normalization_1 (Batch (None, 100)               400       
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 268,210
Trainable params: 267,410
Non-trainable params: 800
_________________________________________________________________
None

通过批过一化后，我们可以看到它的识别准确率比之前都得到了提升。

dropout

有关dropout的详细内容请参考Tensorflow深度学习算法整理

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 归一化过程中，需要将三维图片数据转成二维数据[None, 28, 28] -> [None, 784]
    # 归一化结束之后再转回三维图片数据,这里fit_transform会记录训练数据集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 这里验证数据集会采用训练数据集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        # 批归一化
        keras.layers.BatchNormalization(),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.BatchNormalization(),
        # AlphaDropout，1均值和方差不变，2激活值归一化性质不变
        keras.layers.AlphaDropout(rate=0.5),
        # keras.layers.Dropout(rate=0.5),
        keras.layers.Dense(10, activation='softmax')
        # 上面是将批归一化放在激活函数之后，也可以放在激活函数之前
        # keras.layers.Dense(300),
        # keras.layers.BatchNormalization(),
        # keras.layers.Activation('relu'),
    ])
    # 使用交叉熵损失函数，随机梯度下降法进行训练
    # 这里需要说明的是如果是categorical_crossentropy
    # 则需要将标签数据集进行one_hot处理，如果是sparse_categorical_crossentropy，
    # 则不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我们这里使用三个回调函数Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以保存我们在训练中的指标，比如损失函数值以及识别准确率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是保存模型参数
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前终止训练
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=20,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

运行结果

Train on 55000 samples, validate on 5000 samples
Epoch 1/20
   32/55000 [..............................] - ETA: 32:30 - loss: 2.7970 - accuracy: 0.18752021-10-20 23:56:36.685505: I tensorflow/core/profiler/lib/profiler_session.cc:184] Profiler session started.
55000/55000 [==============================] - 14s 252us/sample - loss: 1.6393 - accuracy: 0.4541 - val_loss: 0.7376 - val_accuracy: 0.7648
Epoch 2/20
55000/55000 [==============================] - 12s 222us/sample - loss: 1.1364 - accuracy: 0.6211 - val_loss: 0.6595 - val_accuracy: 0.7934
Epoch 3/20
55000/55000 [==============================] - 15s 275us/sample - loss: 0.9986 - accuracy: 0.6656 - val_loss: 0.6102 - val_accuracy: 0.8088
Epoch 4/20
55000/55000 [==============================] - 14s 257us/sample - loss: 0.9116 - accuracy: 0.6943 - val_loss: 0.5842 - val_accuracy: 0.8180
Epoch 5/20
55000/55000 [==============================] - 13s 232us/sample - loss: 0.8560 - accuracy: 0.7124 - val_loss: 0.5690 - val_accuracy: 0.8248
Epoch 6/20
55000/55000 [==============================] - 12s 227us/sample - loss: 0.8100 - accuracy: 0.7288 - val_loss: 0.5518 - val_accuracy: 0.8328
Epoch 7/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.7779 - accuracy: 0.7401 - val_loss: 0.5428 - val_accuracy: 0.8372
Epoch 8/20
55000/55000 [==============================] - 13s 229us/sample - loss: 0.7481 - accuracy: 0.7501 - val_loss: 0.5353 - val_accuracy: 0.8414
Epoch 9/20
55000/55000 [==============================] - 12s 225us/sample - loss: 0.7238 - accuracy: 0.7579 - val_loss: 0.5326 - val_accuracy: 0.8420
Epoch 10/20
55000/55000 [==============================] - 12s 222us/sample - loss: 0.6993 - accuracy: 0.7648 - val_loss: 0.5207 - val_accuracy: 0.8458
Epoch 11/20
55000/55000 [==============================] - 12s 222us/sample - loss: 0.6878 - accuracy: 0.7677 - val_loss: 0.5111 - val_accuracy: 0.8478
Epoch 12/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.6640 - accuracy: 0.7769 - val_loss: 0.5103 - val_accuracy: 0.8474
Epoch 13/20
55000/55000 [==============================] - 13s 243us/sample - loss: 0.6516 - accuracy: 0.7794 - val_loss: 0.5073 - val_accuracy: 0.8494
Epoch 14/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.6374 - accuracy: 0.7845 - val_loss: 0.5106 - val_accuracy: 0.8528
Epoch 15/20
55000/55000 [==============================] - 13s 234us/sample - loss: 0.6228 - accuracy: 0.7924 - val_loss: 0.5013 - val_accuracy: 0.8534
Epoch 16/20
55000/55000 [==============================] - 13s 231us/sample - loss: 0.6058 - accuracy: 0.7943 - val_loss: 0.4987 - val_accuracy: 0.8558
Epoch 17/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.5989 - accuracy: 0.7980 - val_loss: 0.4985 - val_accuracy: 0.8556
Epoch 18/20
55000/55000 [==============================] - 13s 232us/sample - loss: 0.5835 - accuracy: 0.8036 - val_loss: 0.4997 - val_accuracy: 0.8566
Epoch 19/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.5795 - accuracy: 0.8058 - val_loss: 0.4907 - val_accuracy: 0.8596
Epoch 20/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.5730 - accuracy: 0.8054 - val_loss: 0.4887 - val_accuracy: 0.8606
10000/1 [================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 1s 81us/sample - loss: 0.4401 - accuracy: 0.8478
[0.564703483581543, 0.8478]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
batch_normalization (BatchNo (None, 300)               1200      
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
batch_normalization_1 (Batch (None, 100)               400       
_________________________________________________________________
alpha_dropout (AlphaDropout) (None, 100)               0         
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 268,210
Trainable params: 267,410
Non-trainable params: 800
_________________________________________________________________
None

通过对dropout的使用，我们发现这个数据集的过拟合现象并不严重，它的识别准确率比之前反而有所降低。

wide_deep模型

google16年发布，可用于分类和回归

稀疏特征

• 离散值特征，比如说一个人的性别信息，只能从两个值里面去进行选择。即类别类型的特征。
• 可以使用One-hot表示。

比如：专业={计算机,人文,其他}。人文=[0, 1, 0]

比如:  词表={人工智能,你,他,网络,...}，他=[0, 0, 1, 0, ...]

叉乘={(计算机,人工智能),(计算机,你),...}，这里表示"专业"中的每个词跟"词表"中的每个词进行组合。叉乘是可以用来刻画一个样本的。

比如说一个物体的所有特征，我们都把它表示成离散特征，我们就可以用叉乘来精确的刻画一个样本，叉乘可以精确的刻画到所有样本的所有可能性。

• 叉乘之后
  • 稀疏特征做叉乘获取共现信息
  • 实现记忆的效果

如果一个物体可以用离散特征去表达它的全部信息，那么这个物体的总量也是确定的，那么叉乘之后的总的集合包含了所有物体的所有可能性。设想我们有一种非常大的模型，这个模型可以把所有样本都记住。当我们遇到新的样本的时候，那么这个样本肯定也是这个大集合中的一个。所以如果我们的模型能够记住大集合中的每一个样本，那么对于任何新的样本它都可以100%的去进行正确的预测。

稀疏特征——优缺点

• 优点
  • 有效，广泛用于工业界
• 缺点
  • 需要人工设计
  • 可能过拟合，所有特征都叉乘，相当于记住每一个样本
  • 泛化能力差，没出现过就不会起效果

密集特征

• 向量表达

比如：词表={人工智能,你,他,网络,...}，他=[0.3, 0.2, 0.6, (n维向量)]，我们可以用向量之间的距离来表示这两个词对应的距离。

• Word2vec工具

可以参考Tensorflow深度学习算法整理(二) 中的内容

密集特征的优缺点

• 优点
  • 带有语义信息，不同向量之间有相关性
  • 兼容没有出现过的特征组合
  • 更少人工参与
• 缺点
  • 过度泛化，推荐不怎么相关的产品

Wide & Deep模型结构

Wide & Deep Vs Wide

在左边的Wide模型中，我们可以看到它只有一层，输入就是稀疏特征，也就是一个One-hot表达。One-hot表达的每个分量都直接连接到输出单元上去。在右边的Wide & Deep模型，它的左半部分也就是一个Wide模型，它的右半部分是一个Deep模型，这个Deep模型就是有多层的神经网络，在这里，我们将输入的数据表现为一个密集的向量表达，在密集表达之上有一个多层的神经网络，在经过多层神经网络之后再连接到输出的神经单元上去。

Wide & Deep Vs Deep

在这里，右图是一个Deep模型，它跟Wide & Deep模型的区别就在于它没有连接稀疏特征的那一块。

这是google商店中一个推荐算法的模型图，它就是一个典型的Wide & Deep模型，它有很多的特征，其中用户已经安装的软件和展示的软件做了叉乘，作为Wide模型的输入。而其他的特征，有些做了Embedding，有些没有做Embedding，把它们拼接起来，作为一个总的特征(密集表示)，再经过多层的神经网络，形成一个Deep的部分，然后再使用一个逻辑回归的损失函数把Wide和Deep给组合起来。现在我们先来看一个加利福尼亚的一个房价的回归问题

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split

if __name__ == "__main__":

    housing = fetch_california_housing()
    print(housing.DESCR)
    print(housing.data.shape)
    print(housing.target.shape)
    pprint.pprint(housing.data[0:5])
    pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    print(X_train.shape, y_train.shape)
    print(X_valid.shape, y_valid.shape)
    print(X_test.shape, y_test.shape)

运行结果

.. _california_housing_dataset:

California Housing dataset
--------------------------

**Data Set Characteristics:**

    :Number of Instances: 20640

    :Number of Attributes: 8 numeric, predictive attributes and the target

    :Attribute Information:
        - MedInc        median income in block
        - HouseAge      median house age in block
        - AveRooms      average number of rooms
        - AveBedrms     average number of bedrooms
        - Population    block population
        - AveOccup      average house occupancy
        - Latitude      house block latitude
        - Longitude     house block longitude

    :Missing Attribute Values: None

This dataset was obtained from the StatLib repository.
http://lib.stat.cmu.edu/datasets/

The target variable is the median house value for California districts.

This dataset was derived from the 1990 U.S. census, using one row per census
block group. A block group is the smallest geographical unit for which the U.S.
Census Bureau publishes sample data (a block group typically has a population
of 600 to 3,000 people).

It can be downloaded/loaded using the
:func:`sklearn.datasets.fetch_california_housing` function.

.. topic:: References

    - Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,
      Statistics and Probability Letters, 33 (1997) 291-297

(20640, 8)
(20640,)
array([[ 8.32520000e+00,  4.10000000e+01,  6.98412698e+00,
         1.02380952e+00,  3.22000000e+02,  2.55555556e+00,
         3.78800000e+01, -1.22230000e+02],
       [ 8.30140000e+00,  2.10000000e+01,  6.23813708e+00,
         9.71880492e-01,  2.40100000e+03,  2.10984183e+00,
         3.78600000e+01, -1.22220000e+02],
       [ 7.25740000e+00,  5.20000000e+01,  8.28813559e+00,
         1.07344633e+00,  4.96000000e+02,  2.80225989e+00,
         3.78500000e+01, -1.22240000e+02],
       [ 5.64310000e+00,  5.20000000e+01,  5.81735160e+00,
         1.07305936e+00,  5.58000000e+02,  2.54794521e+00,
         3.78500000e+01, -1.22250000e+02],
       [ 3.84620000e+00,  5.20000000e+01,  6.28185328e+00,
         1.08108108e+00,  5.65000000e+02,  2.18146718e+00,
         3.78500000e+01, -1.22250000e+02]])
array([4.526, 3.585, 3.521, 3.413, 3.422])
(11610, 8) (11610,)
(3870, 8) (3870,)
(5160, 8) (5160,)

我们可以看到这里有20640条样本数据，维度为8，说明影响房价的有8个特征。其中，我们打印了5条样本数据和标签数据。并将其拆分为训练数据集，验证数据集和测试数据集。现在我们来进行模型训练和打印学习曲线

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 归一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    model = keras.models.Sequential([
        layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
        layers.Dense(1)
    ])
    print(model.summary())
    # 使用均方误差损失函数和随机梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.01))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 学习曲线
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, 30)                270       
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 31        
=================================================================
Total params: 301
Trainable params: 301
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 88us/sample - loss: 1.3027 - val_loss: 2.2434
Epoch 2/100
11610/11610 [==============================] - 1s 45us/sample - loss: 4.6891 - val_loss: 0.4434
Epoch 3/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.4092 - val_loss: 0.3887
Epoch 4/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3811 - val_loss: 0.3797
Epoch 5/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3718 - val_loss: 0.3839
Epoch 6/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3652 - val_loss: 0.3714
Epoch 7/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3620 - val_loss: 0.3662
Epoch 8/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3587 - val_loss: 0.3715
Epoch 9/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3555 - val_loss: 0.3632
Epoch 10/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3525 - val_loss: 0.3620
Epoch 11/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3518 - val_loss: 0.3566
Epoch 12/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3486 - val_loss: 0.3544
Epoch 13/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3466 - val_loss: 0.3494
Epoch 14/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3468 - val_loss: 0.3458
Epoch 15/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3417 - val_loss: 0.3487
Epoch 16/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3412 - val_loss: 0.3449
Epoch 17/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3386 - val_loss: 0.3592
Epoch 18/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3372 - val_loss: 0.3412
Epoch 19/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3364 - val_loss: 0.3426
Epoch 20/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3336 - val_loss: 0.3462
Epoch 21/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3317 - val_loss: 0.3388
Epoch 22/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3300 - val_loss: 0.3461
Epoch 23/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3292 - val_loss: 0.3355
Epoch 24/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3281 - val_loss: 0.3344
Epoch 25/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3266 - val_loss: 0.3374
Epoch 26/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3283 - val_loss: 0.3348
Epoch 27/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3242 - val_loss: 0.3395
Epoch 28/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3247 - val_loss: 0.3322
Epoch 29/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3233 - val_loss: 0.3288
Epoch 30/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3216 - val_loss: 0.3353
Epoch 31/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3216 - val_loss: 0.3293
Epoch 32/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3214 - val_loss: 0.3312
Epoch 33/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3201 - val_loss: 0.3290
Epoch 34/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3189 - val_loss: 0.3319
5160/1 [==================================================] - 0s 19us/sample - loss: 0.4063
0.342487285580746

这里我们发现，我们设置了100次的迭代，但是只到了34次并没有到100次就停止了，再根据学习曲线的情况，我们发现是EarlyStopping的回调函数起到了作用，是梯度下降已经到了瓶颈，趋于平缓。而测试数据集的损失函数值为0.342。

现在我们来将这个简单的回归模型改成Wide & Deep模型

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 归一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函数式API Deep模型
    input = layers.Input(shape=X_train.shape[1:])
    hidden1 = layers.Dense(30, activation='relu')(input)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # input是Wide模型的输入，hidden2是Deep模型的输出
    concat = layers.concatenate([input, hidden2])
    output = layers.Dense(1)(concat)
    model = models.Model(inputs=[input], outputs=[output])
    print(model.summary())
    # 使用均方误差损失函数和随机梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 学习曲线
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 8)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           270         input_1[0][0]                    
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 38)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            39          concatenate[0][0]                
==================================================================================================
Total params: 1,239
Trainable params: 1,239
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 107us/sample - loss: 1.7756 - val_loss: 0.8341
Epoch 2/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.7057 - val_loss: 0.7208
Epoch 3/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.6495 - val_loss: 0.6740
Epoch 4/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.6138 - val_loss: 0.6451
Epoch 5/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5884 - val_loss: 0.6189
Epoch 6/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.5682 - val_loss: 0.6013
Epoch 7/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5515 - val_loss: 0.5818
Epoch 8/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.5387 - val_loss: 0.5675
Epoch 9/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5272 - val_loss: 0.5547
Epoch 10/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5180 - val_loss: 0.5441
Epoch 11/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5085 - val_loss: 0.5336
Epoch 12/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5012 - val_loss: 0.5285
Epoch 13/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4931 - val_loss: 0.5167
Epoch 14/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4881 - val_loss: 0.5128
Epoch 15/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4813 - val_loss: 0.5043
Epoch 16/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4760 - val_loss: 0.4983
Epoch 17/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4710 - val_loss: 0.4928
Epoch 18/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4655 - val_loss: 0.4917
Epoch 19/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4621 - val_loss: 0.4818
Epoch 20/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4571 - val_loss: 0.4776
Epoch 21/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4531 - val_loss: 0.4779
Epoch 22/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4501 - val_loss: 0.4696
Epoch 23/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4455 - val_loss: 0.4650
Epoch 24/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4425 - val_loss: 0.4612
Epoch 25/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4404 - val_loss: 0.4592
Epoch 26/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4367 - val_loss: 0.4542
Epoch 27/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4336 - val_loss: 0.4507
Epoch 28/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4310 - val_loss: 0.4492
Epoch 29/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4278 - val_loss: 0.4470
Epoch 30/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4257 - val_loss: 0.4429
Epoch 31/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4227 - val_loss: 0.4398
Epoch 32/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4208 - val_loss: 0.4375
Epoch 33/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4183 - val_loss: 0.4346
Epoch 34/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4162 - val_loss: 0.4327
Epoch 35/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4136 - val_loss: 0.4306
Epoch 36/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4116 - val_loss: 0.4275
Epoch 37/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4096 - val_loss: 0.4258
Epoch 38/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4075 - val_loss: 0.4236
Epoch 39/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4053 - val_loss: 0.4213
Epoch 40/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4037 - val_loss: 0.4191
Epoch 41/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4016 - val_loss: 0.4180
Epoch 42/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4003 - val_loss: 0.4157
Epoch 43/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3981 - val_loss: 0.4129
Epoch 44/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3962 - val_loss: 0.4130
Epoch 45/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3949 - val_loss: 0.4094
Epoch 46/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3929 - val_loss: 0.4080
Epoch 47/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3915 - val_loss: 0.4067
Epoch 48/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3898 - val_loss: 0.4044
Epoch 49/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3885 - val_loss: 0.4032
Epoch 50/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3868 - val_loss: 0.4019
Epoch 51/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3850 - val_loss: 0.3995
Epoch 52/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3839 - val_loss: 0.3993
Epoch 53/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3825 - val_loss: 0.3971
Epoch 54/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3809 - val_loss: 0.3951
Epoch 55/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3798 - val_loss: 0.3940
Epoch 56/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3788 - val_loss: 0.3929
Epoch 57/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3775 - val_loss: 0.3916
Epoch 58/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3760 - val_loss: 0.3906
Epoch 59/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3749 - val_loss: 0.3900
Epoch 60/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3742 - val_loss: 0.3880
Epoch 61/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3726 - val_loss: 0.3866
Epoch 62/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3718 - val_loss: 0.3863
Epoch 63/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3706 - val_loss: 0.3853
Epoch 64/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.3698 - val_loss: 0.3834
Epoch 65/100
11610/11610 [==============================] - 1s 71us/sample - loss: 0.3687 - val_loss: 0.3834
Epoch 66/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3679 - val_loss: 0.3818
Epoch 67/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3668 - val_loss: 0.3811
Epoch 68/100
11610/11610 [==============================] - 1s 78us/sample - loss: 0.3658 - val_loss: 0.3801
Epoch 69/100
11610/11610 [==============================] - 1s 76us/sample - loss: 0.3649 - val_loss: 0.3787
Epoch 70/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.3639 - val_loss: 0.3781
Epoch 71/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3634 - val_loss: 0.3769
Epoch 72/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3621 - val_loss: 0.3762
Epoch 73/100
11610/11610 [==============================] - 1s 66us/sample - loss: 0.3615 - val_loss: 0.3762
Epoch 74/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3606 - val_loss: 0.3746
Epoch 75/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3596 - val_loss: 0.3749
Epoch 76/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3590 - val_loss: 0.3736
Epoch 77/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3580 - val_loss: 0.3740
Epoch 78/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3576 - val_loss: 0.3721
Epoch 79/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3562 - val_loss: 0.3712
Epoch 80/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3558 - val_loss: 0.3710
Epoch 81/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3549 - val_loss: 0.3710
Epoch 82/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3541 - val_loss: 0.3707
Epoch 83/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3537 - val_loss: 0.3690
Epoch 84/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3530 - val_loss: 0.3691
Epoch 85/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3520 - val_loss: 0.3679
Epoch 86/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3514 - val_loss: 0.3673
Epoch 87/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3508 - val_loss: 0.3671
Epoch 88/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3500 - val_loss: 0.3662
Epoch 89/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3496 - val_loss: 0.3655
Epoch 90/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3487 - val_loss: 0.3661
Epoch 91/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3484 - val_loss: 0.3658
Epoch 92/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3479 - val_loss: 0.3641
Epoch 93/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3468 - val_loss: 0.3637
Epoch 94/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3462 - val_loss: 0.3640
Epoch 95/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3456 - val_loss: 0.3629
Epoch 96/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3449 - val_loss: 0.3621
Epoch 97/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3444 - val_loss: 0.3621
Epoch 98/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3431 - val_loss: 0.3622
Epoch 99/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3434 - val_loss: 0.3606
Epoch 100/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3430 - val_loss: 0.3604
5160/1 [========================================================] - 0s 24us/sample - loss: 0.4557
0.36289026386053985

在这里，我们使用的是函数式编程来实现的Wide & Deep模型，下面我们来使用子类API来实现这个模型

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 归一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函数式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的输入，hidden2是Deep模型的输出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子类API，父类为keras.models.Model
    class WideDeepModel(models.Model):

        def __init__(self):
            super(WideDeepModel, self).__init__()
            # 定义模型的层次
            self.hidden1_layer = layers.Dense(30, activation='relu')
            self.hidden2_layer = layers.Dense(30, activation='relu')
            self.output_layer = layers.Dense(1)


        def call(self, input):
            # 完成模型的正向计算
            hidden1 = self.hidden1_layer(input)
            hidden2 = self.hidden2_layer(hidden1)
            concat = layers.concatenate([input, hidden2])
            output = self.output_layer(concat)
            return output

    model = WideDeepModel()
    # 也可以使用层堆叠
    # model = models.Sequential([
    #     WideDeepModel()
    # ])
    model.build(input_shape=(None, 8))
    print(model.summary())
    # 使用均方误差损失函数和随机梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 学习曲线
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Model: "wide_deep_model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                multiple                  270       
_________________________________________________________________
dense_1 (Dense)              multiple                  930       
_________________________________________________________________
dense_2 (Dense)              multiple                  39        
=================================================================
Total params: 1,239
Trainable params: 1,239
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 107us/sample - loss: 2.0999 - val_loss: 0.8217
Epoch 2/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.6996 - val_loss: 0.7236
Epoch 3/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.6376 - val_loss: 0.6704
Epoch 4/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.6003 - val_loss: 0.6322
Epoch 5/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5720 - val_loss: 0.6104
Epoch 6/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5499 - val_loss: 0.5869
Epoch 7/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5324 - val_loss: 0.5653
Epoch 8/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.5188 - val_loss: 0.5499
Epoch 9/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5075 - val_loss: 0.5401
Epoch 10/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4970 - val_loss: 0.5268
Epoch 11/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4902 - val_loss: 0.5204
Epoch 12/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.4819 - val_loss: 0.5103
Epoch 13/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.4759 - val_loss: 0.5056
Epoch 14/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.4697 - val_loss: 0.4977
Epoch 15/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.4654 - val_loss: 0.4931
Epoch 16/100
11610/11610 [==============================] - 1s 83us/sample - loss: 0.4597 - val_loss: 0.4928
Epoch 17/100
11610/11610 [==============================] - 1s 78us/sample - loss: 0.4538 - val_loss: 0.4821
Epoch 18/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.4506 - val_loss: 0.4765
Epoch 19/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.4465 - val_loss: 0.4744
Epoch 20/100
11610/11610 [==============================] - 1s 58us/sample - loss: 0.4430 - val_loss: 0.4762
Epoch 21/100
11610/11610 [==============================] - 1s 99us/sample - loss: 0.4391 - val_loss: 0.4668
Epoch 22/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.4361 - val_loss: 0.4641
Epoch 23/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.4328 - val_loss: 0.4587
Epoch 24/100
11610/11610 [==============================] - 1s 66us/sample - loss: 0.4293 - val_loss: 0.4565
Epoch 25/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4266 - val_loss: 0.4525
Epoch 26/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4237 - val_loss: 0.4481
Epoch 27/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4210 - val_loss: 0.4452
Epoch 28/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4178 - val_loss: 0.4426
Epoch 29/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.4163 - val_loss: 0.4410
Epoch 30/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.4135 - val_loss: 0.4372
Epoch 31/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.4111 - val_loss: 0.4350
Epoch 32/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4096 - val_loss: 0.4329
Epoch 33/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4075 - val_loss: 0.4298
Epoch 34/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4053 - val_loss: 0.4293
Epoch 35/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4035 - val_loss: 0.4263
Epoch 36/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4018 - val_loss: 0.4236
Epoch 37/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3997 - val_loss: 0.4221
Epoch 38/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3977 - val_loss: 0.4207
Epoch 39/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3962 - val_loss: 0.4175
Epoch 40/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3947 - val_loss: 0.4152
Epoch 41/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3927 - val_loss: 0.4140
Epoch 42/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3914 - val_loss: 0.4131
Epoch 43/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3897 - val_loss: 0.4102
Epoch 44/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3883 - val_loss: 0.4088
Epoch 45/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3869 - val_loss: 0.4063
Epoch 46/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3858 - val_loss: 0.4053
Epoch 47/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3841 - val_loss: 0.4041
Epoch 48/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3832 - val_loss: 0.4029
Epoch 49/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3818 - val_loss: 0.4002
Epoch 50/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.3807 - val_loss: 0.3987
Epoch 51/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3791 - val_loss: 0.3970
Epoch 52/100
11610/11610 [==============================] - 1s 73us/sample - loss: 0.3779 - val_loss: 0.3970
Epoch 53/100
11610/11610 [==============================] - 1s 98us/sample - loss: 0.3768 - val_loss: 0.3964
Epoch 54/100
11610/11610 [==============================] - 1s 92us/sample - loss: 0.3756 - val_loss: 0.3941
Epoch 55/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3746 - val_loss: 0.3926
Epoch 56/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3739 - val_loss: 0.3914
Epoch 57/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3727 - val_loss: 0.3896
Epoch 58/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3718 - val_loss: 0.3879
Epoch 59/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3701 - val_loss: 0.3900
Epoch 60/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3696 - val_loss: 0.3865
Epoch 61/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3683 - val_loss: 0.3855
Epoch 62/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3674 - val_loss: 0.3860
Epoch 63/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3669 - val_loss: 0.3831
Epoch 64/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3652 - val_loss: 0.3820
Epoch 65/100
11610/11610 [==============================] - 1s 76us/sample - loss: 0.3645 - val_loss: 0.3818
Epoch 66/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.3641 - val_loss: 0.3794
Epoch 67/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.3623 - val_loss: 0.3799
Epoch 68/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.3623 - val_loss: 0.3782
Epoch 69/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3606 - val_loss: 0.3766
Epoch 70/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.3599 - val_loss: 0.3766
Epoch 71/100
11610/11610 [==============================] - 1s 87us/sample - loss: 0.3589 - val_loss: 0.3748
Epoch 72/100
11610/11610 [==============================] - 1s 100us/sample - loss: 0.3582 - val_loss: 0.3743
Epoch 73/100
11610/11610 [==============================] - 1s 60us/sample - loss: 0.3573 - val_loss: 0.3728
Epoch 74/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3564 - val_loss: 0.3735
Epoch 75/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.3562 - val_loss: 0.3718
Epoch 76/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.3557 - val_loss: 0.3714
Epoch 77/100
11610/11610 [==============================] - 1s 85us/sample - loss: 0.3548 - val_loss: 0.3703
Epoch 78/100
11610/11610 [==============================] - 1s 93us/sample - loss: 0.3540 - val_loss: 0.3700
Epoch 79/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3528 - val_loss: 0.3685
Epoch 80/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3526 - val_loss: 0.3678
Epoch 81/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3512 - val_loss: 0.3678
Epoch 82/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3506 - val_loss: 0.3672
Epoch 83/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3508 - val_loss: 0.3657
Epoch 84/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3495 - val_loss: 0.3669
Epoch 85/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3491 - val_loss: 0.3670
Epoch 86/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3490 - val_loss: 0.3642
Epoch 87/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3486 - val_loss: 0.3642
Epoch 88/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3478 - val_loss: 0.3643
Epoch 89/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3474 - val_loss: 0.3631
Epoch 90/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3467 - val_loss: 0.3614
Epoch 91/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3451 - val_loss: 0.3604
Epoch 92/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3444 - val_loss: 0.3606
Epoch 93/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3436 - val_loss: 0.3592
Epoch 94/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3435 - val_loss: 0.3588
Epoch 95/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3585
Epoch 96/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3576
Epoch 97/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3578
Epoch 98/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3419 - val_loss: 0.3569
Epoch 99/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3412 - val_loss: 0.3572
Epoch 100/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3404 - val_loss: 0.3551
5160/1 [===========================================================] - 0s 45us/sample - loss: 0.4603
0.36134041002554484

由于Wide & Deep模型可以分别挑选特征来进行输入，现在我们来改成多输入的模式

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 归一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函数式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的输入，hidden2是Deep模型的输出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子类API，父类为keras.models.Model
    # class WideDeepModel(models.Model):
    #
    #     def __init__(self):
    #         super(WideDeepModel, self).__init__()
    #         # 定义模型的层次
    #         self.hidden1_layer = layers.Dense(30, activation='relu')
    #         self.hidden2_layer = layers.Dense(30, activation='relu')
    #         self.output_layer = layers.Dense(1)
    #
    #
    #     def call(self, input):
    #         # 完成模型的正向计算
    #         hidden1 = self.hidden1_layer(input)
    #         hidden2 = self.hidden2_layer(hidden1)
    #         concat = layers.concatenate([input, hidden2])
    #         output = self.output_layer(concat)
    #         return output
    #
    # model = WideDeepModel()
    # # 也可以使用层堆叠
    # # model = models.Sequential([
    # #     WideDeepModel()
    # # ])
    # model.build(input_shape=(None, 8))
    # 多输入 wide模型输入前5个特征，deep模型输入后6个特征
    input_wide = layers.Input(shape=[5])
    input_deep = layers.Input(shape=[6])
    hidden1 = layers.Dense(30, activation='relu')(input_deep)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    concat = layers.concatenate([input_wide, hidden2])
    output = layers.Dense(1)(concat)
    model = models.Model(inputs=[input_wide, input_deep], outputs=[output])
    print(model.summary())
    # 使用均方误差损失函数和随机梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    X_train_wide = X_train[:, :5]
    X_train_deep = X_train[:, 2:]
    X_valid_wide = X_valid[:, :5]
    X_valid_deep = X_valid[:, 2:]
    X_test_wide = X_test[:, :5]
    X_test_deep = X_test[:, 2:]
    history = model.fit([X_train_wide, X_train_deep], y_train,
                        validation_data=([X_valid_wide, X_valid_deep], y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate([X_test_wide, X_test_deep], y_test))

    def plot_learning_curves(history):
        # 学习曲线
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 6)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           210         input_2[0][0]                    
__________________________________________________________________________________________________
input_1 (InputLayer)            [(None, 5)]          0                                            
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 35)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            36          concatenate[0][0]                
==================================================================================================
Total params: 1,176
Trainable params: 1,176
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 2s 138us/sample - loss: 1.9978 - val_loss: 0.9837
Epoch 2/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.8110 - val_loss: 0.7862
Epoch 3/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.7007 - val_loss: 0.7168
Epoch 4/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.6489 - val_loss: 0.6742
Epoch 5/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.6118 - val_loss: 0.6378
Epoch 6/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5831 - val_loss: 0.6127
Epoch 7/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5607 - val_loss: 0.5910
Epoch 8/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5427 - val_loss: 0.5736
Epoch 9/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.5275 - val_loss: 0.5589
Epoch 10/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5144 - val_loss: 0.5455
Epoch 11/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5033 - val_loss: 0.5353
Epoch 12/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4939 - val_loss: 0.5258
Epoch 13/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4853 - val_loss: 0.5156
Epoch 14/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4784 - val_loss: 0.5084
Epoch 15/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4718 - val_loss: 0.5035
Epoch 16/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4657 - val_loss: 0.4952
Epoch 17/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4606 - val_loss: 0.4887
Epoch 18/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4560 - val_loss: 0.4840
Epoch 19/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4517 - val_loss: 0.4805
Epoch 20/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4477 - val_loss: 0.4773
Epoch 21/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4451 - val_loss: 0.4727
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4423 - val_loss: 0.4697
Epoch 23/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4397 - val_loss: 0.4708
Epoch 24/100
11610/11610 [==============================] - 1s 72us/sample - loss: 0.4375 - val_loss: 0.4650
Epoch 25/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4350 - val_loss: 0.4633
Epoch 26/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4329 - val_loss: 0.4593
Epoch 27/100
11610/11610 [==============================] - 1s 88us/sample - loss: 0.4310 - val_loss: 0.4569
Epoch 28/100
11610/11610 [==============================] - 1s 99us/sample - loss: 0.4294 - val_loss: 0.4558
Epoch 29/100
11610/11610 [==============================] - 1s 97us/sample - loss: 0.4276 - val_loss: 0.4521
Epoch 30/100
11610/11610 [==============================] - 1s 120us/sample - loss: 0.4254 - val_loss: 0.4494
Epoch 31/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4239 - val_loss: 0.4484
Epoch 32/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.4219 - val_loss: 0.4470
Epoch 33/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4209 - val_loss: 0.4456
Epoch 34/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4194 - val_loss: 0.4433
Epoch 35/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4179 - val_loss: 0.4427
Epoch 36/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4163 - val_loss: 0.4399
Epoch 37/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4150 - val_loss: 0.4399
Epoch 38/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4137 - val_loss: 0.4370
Epoch 39/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4122 - val_loss: 0.4364
Epoch 40/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4112 - val_loss: 0.4369
Epoch 41/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4098 - val_loss: 0.4329
Epoch 42/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4088 - val_loss: 0.4326
Epoch 43/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4077 - val_loss: 0.4299
Epoch 44/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4066 - val_loss: 0.4304
Epoch 45/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4054 - val_loss: 0.4284
Epoch 46/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.4043 - val_loss: 0.4286
Epoch 47/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.4036 - val_loss: 0.4263
Epoch 48/100
11610/11610 [==============================] - 1s 88us/sample - loss: 0.4026 - val_loss: 0.4251
Epoch 49/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.4014 - val_loss: 0.4240
Epoch 50/100
11610/11610 [==============================] - 1s 96us/sample - loss: 0.4007 - val_loss: 0.4231
Epoch 51/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.3998 - val_loss: 0.4216
Epoch 52/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3990 - val_loss: 0.4235
Epoch 53/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3981 - val_loss: 0.4211
Epoch 54/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3976 - val_loss: 0.4210
Epoch 55/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3965 - val_loss: 0.4217
Epoch 56/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3959 - val_loss: 0.4174
Epoch 57/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3953 - val_loss: 0.4175
Epoch 58/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.3943 - val_loss: 0.4163
Epoch 59/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3935 - val_loss: 0.4156
Epoch 60/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3928 - val_loss: 0.4150
Epoch 61/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3922 - val_loss: 0.4147
Epoch 62/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3913 - val_loss: 0.4137
Epoch 63/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3906 - val_loss: 0.4128
Epoch 64/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3898 - val_loss: 0.4113
Epoch 65/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3894 - val_loss: 0.4121
Epoch 66/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3884 - val_loss: 0.4110
Epoch 67/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3878 - val_loss: 0.4129
Epoch 68/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3873 - val_loss: 0.4109
Epoch 69/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3864 - val_loss: 0.4101
Epoch 70/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3858 - val_loss: 0.4092
Epoch 71/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3853 - val_loss: 0.4073
Epoch 72/100
11610/11610 [==============================] - 1s 94us/sample - loss: 0.3847 - val_loss: 0.4076
Epoch 73/100
11610/11610 [==============================] - 1s 98us/sample - loss: 0.3842 - val_loss: 0.4059
Epoch 74/100
11610/11610 [==============================] - 1s 80us/sample - loss: 0.3834 - val_loss: 0.4047
Epoch 75/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3826 - val_loss: 0.4056
Epoch 76/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.3822 - val_loss: 0.4031
Epoch 77/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3814 - val_loss: 0.4049
Epoch 78/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3812 - val_loss: 0.4029
Epoch 79/100
11610/11610 [==============================] - 1s 59us/sample - loss: 0.3800 - val_loss: 0.4022
Epoch 80/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3794 - val_loss: 0.4004
Epoch 81/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3793 - val_loss: 0.4004
Epoch 82/100
11610/11610 [==============================] - 1s 79us/sample - loss: 0.3784 - val_loss: 0.3998
Epoch 83/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3778 - val_loss: 0.3988
Epoch 84/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3770 - val_loss: 0.3981
Epoch 85/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3764 - val_loss: 0.3987
Epoch 86/100
11610/11610 [==============================] - 1s 83us/sample - loss: 0.3756 - val_loss: 0.3970
Epoch 87/100
11610/11610 [==============================] - 1s 89us/sample - loss: 0.3751 - val_loss: 0.3969
Epoch 88/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3747 - val_loss: 0.3950
Epoch 89/100
11610/11610 [==============================] - 1s 90us/sample - loss: 0.3740 - val_loss: 0.3951
Epoch 90/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.3733 - val_loss: 0.3945
Epoch 91/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3727 - val_loss: 0.3937
Epoch 92/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3721 - val_loss: 0.3937
Epoch 93/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3717 - val_loss: 0.3932
Epoch 94/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3709 - val_loss: 0.3930
Epoch 95/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3703 - val_loss: 0.3919
Epoch 96/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3695 - val_loss: 0.3921
Epoch 97/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3692 - val_loss: 0.3909
Epoch 98/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3687 - val_loss: 0.3892
Epoch 99/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3676 - val_loss: 0.3902
Epoch 100/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3677 - val_loss: 0.3888
5160/1 [=======================================================] - 0s 24us/sample - loss: 0.4975
0.3788368522658829

有了多输入就会有多输出

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 归一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函数式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的输入，hidden2是Deep模型的输出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子类API，父类为keras.models.Model
    # class WideDeepModel(models.Model):
    #
    #     def __init__(self):
    #         super(WideDeepModel, self).__init__()
    #         # 定义模型的层次
    #         self.hidden1_layer = layers.Dense(30, activation='relu')
    #         self.hidden2_layer = layers.Dense(30, activation='relu')
    #         self.output_layer = layers.Dense(1)
    #
    #
    #     def call(self, input):
    #         # 完成模型的正向计算
    #         hidden1 = self.hidden1_layer(input)
    #         hidden2 = self.hidden2_layer(hidden1)
    #         concat = layers.concatenate([input, hidden2])
    #         output = self.output_layer(concat)
    #         return output
    #
    # model = WideDeepModel()
    # # 也可以使用层堆叠
    # # model = models.Sequential([
    # #     WideDeepModel()
    # # ])
    # model.build(input_shape=(None, 8))
    # 多输入 wide模型输入前5个特征，deep模型输入后6个特征
    input_wide = layers.Input(shape=[5])
    input_deep = layers.Input(shape=[6])
    hidden1 = layers.Dense(30, activation='relu')(input_deep)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    concat = layers.concatenate([input_wide, hidden2])
    # 多输出
    output1 = layers.Dense(1)(concat)
    output2 = layers.Dense(1)(hidden2)
    model = models.Model(inputs=[input_wide, input_deep], outputs=[output1, output2])
    print(model.summary())
    # 使用均方误差损失函数和随机梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    X_train_wide = X_train[:, :5]
    X_train_deep = X_train[:, 2:]
    X_valid_wide = X_valid[:, :5]
    X_valid_deep = X_valid[:, 2:]
    X_test_wide = X_test[:, :5]
    X_test_deep = X_test[:, 2:]
    history = model.fit([X_train_wide, X_train_deep], [y_train, y_train],
                        validation_data=([X_valid_wide, X_valid_deep], [y_valid, y_valid]),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate([X_test_wide, X_test_deep], [y_test, y_test]))

    def plot_learning_curves(history):
        # 学习曲线
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 6)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           210         input_2[0][0]                    
__________________________________________________________________________________________________
input_1 (InputLayer)            [(None, 5)]          0                                            
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 35)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            36          concatenate[0][0]                
__________________________________________________________________________________________________
dense_3 (Dense)                 (None, 1)            31          dense_1[0][0]                    
==================================================================================================
Total params: 1,207
Trainable params: 1,207
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 2s 151us/sample - loss: 4.6904 - dense_2_loss: 2.2887 - dense_3_loss: 2.4007 - val_loss: 2.2365 - val_dense_2_loss: 0.8763 - val_dense_3_loss: 1.3602
Epoch 2/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.8804 - dense_2_loss: 0.7132 - dense_3_loss: 1.1668 - val_loss: 1.8578 - val_dense_2_loss: 0.6893 - val_dense_3_loss: 1.1685
Epoch 3/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.6375 - dense_2_loss: 0.6051 - dense_3_loss: 1.0320 - val_loss: 1.6503 - val_dense_2_loss: 0.6125 - val_dense_3_loss: 1.0377
Epoch 4/100
11610/11610 [==============================] - 1s 63us/sample - loss: 1.4545 - dense_2_loss: 0.5502 - dense_3_loss: 0.9039 - val_loss: 1.4679 - val_dense_2_loss: 0.5670 - val_dense_3_loss: 0.9009
Epoch 5/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.3185 - dense_2_loss: 0.5166 - dense_3_loss: 0.8021 - val_loss: 1.3543 - val_dense_2_loss: 0.5365 - val_dense_3_loss: 0.8177
Epoch 6/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.2300 - dense_2_loss: 0.4951 - dense_3_loss: 0.7349 - val_loss: 1.2700 - val_dense_2_loss: 0.5162 - val_dense_3_loss: 0.7536
Epoch 7/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.1740 - dense_2_loss: 0.4805 - dense_3_loss: 0.6936 - val_loss: 1.2194 - val_dense_2_loss: 0.5032 - val_dense_3_loss: 0.7161
Epoch 8/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.1349 - dense_2_loss: 0.4695 - dense_3_loss: 0.6651 - val_loss: 1.1812 - val_dense_2_loss: 0.4888 - val_dense_3_loss: 0.6923
Epoch 9/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.1066 - dense_2_loss: 0.4612 - dense_3_loss: 0.6452 - val_loss: 1.1580 - val_dense_2_loss: 0.4876 - val_dense_3_loss: 0.6703
Epoch 10/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0847 - dense_2_loss: 0.4549 - dense_3_loss: 0.6300 - val_loss: 1.1370 - val_dense_2_loss: 0.4784 - val_dense_3_loss: 0.6585
Epoch 11/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0667 - dense_2_loss: 0.4483 - dense_3_loss: 0.6183 - val_loss: 1.1159 - val_dense_2_loss: 0.4717 - val_dense_3_loss: 0.6441
Epoch 12/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0524 - dense_2_loss: 0.4440 - dense_3_loss: 0.6086 - val_loss: 1.1014 - val_dense_2_loss: 0.4640 - val_dense_3_loss: 0.6373
Epoch 13/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0388 - dense_2_loss: 0.4397 - dense_3_loss: 0.5995 - val_loss: 1.0919 - val_dense_2_loss: 0.4677 - val_dense_3_loss: 0.6241
Epoch 14/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0277 - dense_2_loss: 0.4357 - dense_3_loss: 0.5921 - val_loss: 1.0758 - val_dense_2_loss: 0.4573 - val_dense_3_loss: 0.6184
Epoch 15/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.0185 - dense_2_loss: 0.4323 - dense_3_loss: 0.5861 - val_loss: 1.0668 - val_dense_2_loss: 0.4580 - val_dense_3_loss: 0.6087
Epoch 16/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0102 - dense_2_loss: 0.4295 - dense_3_loss: 0.5807 - val_loss: 1.0546 - val_dense_2_loss: 0.4518 - val_dense_3_loss: 0.6027
Epoch 17/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0019 - dense_2_loss: 0.4267 - dense_3_loss: 0.5751 - val_loss: 1.0487 - val_dense_2_loss: 0.4512 - val_dense_3_loss: 0.5974
Epoch 18/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.9950 - dense_2_loss: 0.4239 - dense_3_loss: 0.5714 - val_loss: 1.0379 - val_dense_2_loss: 0.4465 - val_dense_3_loss: 0.5913
Epoch 19/100
11610/11610 [==============================] - 1s 109us/sample - loss: 0.9884 - dense_2_loss: 0.4213 - dense_3_loss: 0.5668 - val_loss: 1.0312 - val_dense_2_loss: 0.4447 - val_dense_3_loss: 0.5864
Epoch 20/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.9820 - dense_2_loss: 0.4188 - dense_3_loss: 0.5630 - val_loss: 1.0274 - val_dense_2_loss: 0.4428 - val_dense_3_loss: 0.5844
Epoch 21/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.9757 - dense_2_loss: 0.4165 - dense_3_loss: 0.5590 - val_loss: 1.0197 - val_dense_2_loss: 0.4413 - val_dense_3_loss: 0.5783
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.9690 - dense_2_loss: 0.4140 - dense_3_loss: 0.5547 - val_loss: 1.0125 - val_dense_2_loss: 0.4372 - val_dense_3_loss: 0.5752
Epoch 23/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9633 - dense_2_loss: 0.4119 - dense_3_loss: 0.5514 - val_loss: 1.0073 - val_dense_2_loss: 0.4348 - val_dense_3_loss: 0.5723
Epoch 24/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9581 - dense_2_loss: 0.4105 - dense_3_loss: 0.5483 - val_loss: 1.0053 - val_dense_2_loss: 0.4386 - val_dense_3_loss: 0.5666
Epoch 25/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9543 - dense_2_loss: 0.4085 - dense_3_loss: 0.5457 - val_loss: 0.9971 - val_dense_2_loss: 0.4320 - val_dense_3_loss: 0.5650
Epoch 26/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9473 - dense_2_loss: 0.4054 - dense_3_loss: 0.5417 - val_loss: 0.9948 - val_dense_2_loss: 0.4335 - val_dense_3_loss: 0.5612
Epoch 27/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9427 - dense_2_loss: 0.4044 - dense_3_loss: 0.5383 - val_loss: 0.9884 - val_dense_2_loss: 0.4279 - val_dense_3_loss: 0.5604
Epoch 28/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9392 - dense_2_loss: 0.4028 - dense_3_loss: 0.5362 - val_loss: 0.9836 - val_dense_2_loss: 0.4280 - val_dense_3_loss: 0.5555
Epoch 29/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9333 - dense_2_loss: 0.4003 - dense_3_loss: 0.5329 - val_loss: 0.9803 - val_dense_2_loss: 0.4257 - val_dense_3_loss: 0.5544
Epoch 30/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9297 - dense_2_loss: 0.3990 - dense_3_loss: 0.5304 - val_loss: 0.9735 - val_dense_2_loss: 0.4229 - val_dense_3_loss: 0.5505
Epoch 31/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.9257 - dense_2_loss: 0.3978 - dense_3_loss: 0.5278 - val_loss: 0.9735 - val_dense_2_loss: 0.4216 - val_dense_3_loss: 0.5518
Epoch 32/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9202 - dense_2_loss: 0.3956 - dense_3_loss: 0.5247 - val_loss: 0.9665 - val_dense_2_loss: 0.4209 - val_dense_3_loss: 0.5455
Epoch 33/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9178 - dense_2_loss: 0.3946 - dense_3_loss: 0.5229 - val_loss: 0.9594 - val_dense_2_loss: 0.4180 - val_dense_3_loss: 0.5413
Epoch 34/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9121 - dense_2_loss: 0.3925 - dense_3_loss: 0.5197 - val_loss: 0.9559 - val_dense_2_loss: 0.4147 - val_dense_3_loss: 0.5411
Epoch 35/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9080 - dense_2_loss: 0.3910 - dense_3_loss: 0.5168 - val_loss: 0.9536 - val_dense_2_loss: 0.4119 - val_dense_3_loss: 0.5415
Epoch 36/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9046 - dense_2_loss: 0.3900 - dense_3_loss: 0.5146 - val_loss: 0.9511 - val_dense_2_loss: 0.4139 - val_dense_3_loss: 0.5371
Epoch 37/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9000 - dense_2_loss: 0.3882 - dense_3_loss: 0.5118 - val_loss: 0.9507 - val_dense_2_loss: 0.4199 - val_dense_3_loss: 0.5307
Epoch 38/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8990 - dense_2_loss: 0.3884 - dense_3_loss: 0.5110 - val_loss: 0.9416 - val_dense_2_loss: 0.4145 - val_dense_3_loss: 0.5270
Epoch 39/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8928 - dense_2_loss: 0.3859 - dense_3_loss: 0.5071 - val_loss: 0.9357 - val_dense_2_loss: 0.4092 - val_dense_3_loss: 0.5264
Epoch 40/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8882 - dense_2_loss: 0.3838 - dense_3_loss: 0.5044 - val_loss: 0.9325 - val_dense_2_loss: 0.4092 - val_dense_3_loss: 0.5233
Epoch 41/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8846 - dense_2_loss: 0.3831 - dense_3_loss: 0.5016 - val_loss: 0.9277 - val_dense_2_loss: 0.4055 - val_dense_3_loss: 0.5221
Epoch 42/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8831 - dense_2_loss: 0.3827 - dense_3_loss: 0.5002 - val_loss: 0.9259 - val_dense_2_loss: 0.4053 - val_dense_3_loss: 0.5205
Epoch 43/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8804 - dense_2_loss: 0.3817 - dense_3_loss: 0.4987 - val_loss: 0.9199 - val_dense_2_loss: 0.4035 - val_dense_3_loss: 0.5163
Epoch 44/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8737 - dense_2_loss: 0.3792 - dense_3_loss: 0.4946 - val_loss: 0.9165 - val_dense_2_loss: 0.4009 - val_dense_3_loss: 0.5154
Epoch 45/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8698 - dense_2_loss: 0.3776 - dense_3_loss: 0.4920 - val_loss: 0.9099 - val_dense_2_loss: 0.3978 - val_dense_3_loss: 0.5120
Epoch 46/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8689 - dense_2_loss: 0.3779 - dense_3_loss: 0.4910 - val_loss: 0.9068 - val_dense_2_loss: 0.3983 - val_dense_3_loss: 0.5084
Epoch 47/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8622 - dense_2_loss: 0.3751 - dense_3_loss: 0.4870 - val_loss: 0.9008 - val_dense_2_loss: 0.3938 - val_dense_3_loss: 0.5069
Epoch 48/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8588 - dense_2_loss: 0.3739 - dense_3_loss: 0.4848 - val_loss: 0.8974 - val_dense_2_loss: 0.3932 - val_dense_3_loss: 0.5041
Epoch 49/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8582 - dense_2_loss: 0.3740 - dense_3_loss: 0.4840 - val_loss: 0.8923 - val_dense_2_loss: 0.3919 - val_dense_3_loss: 0.5003
Epoch 50/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8522 - dense_2_loss: 0.3718 - dense_3_loss: 0.4803 - val_loss: 0.8924 - val_dense_2_loss: 0.3922 - val_dense_3_loss: 0.5000
Epoch 51/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.8503 - dense_2_loss: 0.3715 - dense_3_loss: 0.4789 - val_loss: 0.8888 - val_dense_2_loss: 0.3900 - val_dense_3_loss: 0.4986
Epoch 52/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.8487 - dense_2_loss: 0.3714 - dense_3_loss: 0.4772 - val_loss: 0.8834 - val_dense_2_loss: 0.3910 - val_dense_3_loss: 0.4923
Epoch 53/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8443 - dense_2_loss: 0.3696 - dense_3_loss: 0.4746 - val_loss: 0.8799 - val_dense_2_loss: 0.3882 - val_dense_3_loss: 0.4915
Epoch 54/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8423 - dense_2_loss: 0.3691 - dense_3_loss: 0.4731 - val_loss: 0.8745 - val_dense_2_loss: 0.3870 - val_dense_3_loss: 0.4874
Epoch 55/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8385 - dense_2_loss: 0.3679 - dense_3_loss: 0.4705 - val_loss: 0.8729 - val_dense_2_loss: 0.3859 - val_dense_3_loss: 0.4870
Epoch 56/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8307 - dense_2_loss: 0.3647 - dense_3_loss: 0.4659 - val_loss: 0.8685 - val_dense_2_loss: 0.3823 - val_dense_3_loss: 0.4861
Epoch 57/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.8316 - dense_2_loss: 0.3654 - dense_3_loss: 0.4663 - val_loss: 0.8642 - val_dense_2_loss: 0.3818 - val_dense_3_loss: 0.4823
Epoch 58/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8277 - dense_2_loss: 0.3640 - dense_3_loss: 0.4636 - val_loss: 0.8592 - val_dense_2_loss: 0.3810 - val_dense_3_loss: 0.4781
Epoch 59/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8323 - dense_2_loss: 0.3661 - dense_3_loss: 0.4661 - val_loss: 0.8578 - val_dense_2_loss: 0.3809 - val_dense_3_loss: 0.4768
Epoch 60/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8185 - dense_2_loss: 0.3608 - dense_3_loss: 0.4580 - val_loss: 0.8512 - val_dense_2_loss: 0.3777 - val_dense_3_loss: 0.4734
Epoch 61/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8188 - dense_2_loss: 0.3609 - dense_3_loss: 0.4577 - val_loss: 0.8514 - val_dense_2_loss: 0.3779 - val_dense_3_loss: 0.4734
Epoch 62/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8165 - dense_2_loss: 0.3608 - dense_3_loss: 0.4556 - val_loss: 0.8442 - val_dense_2_loss: 0.3746 - val_dense_3_loss: 0.4695
Epoch 63/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.8118 - dense_2_loss: 0.3589 - dense_3_loss: 0.4530 - val_loss: 0.8407 - val_dense_2_loss: 0.3724 - val_dense_3_loss: 0.4682
Epoch 64/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8097 - dense_2_loss: 0.3587 - dense_3_loss: 0.4512 - val_loss: 0.8400 - val_dense_2_loss: 0.3758 - val_dense_3_loss: 0.4640
Epoch 65/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8025 - dense_2_loss: 0.3552 - dense_3_loss: 0.4471 - val_loss: 0.8353 - val_dense_2_loss: 0.3721 - val_dense_3_loss: 0.4631
Epoch 66/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7997 - dense_2_loss: 0.3542 - dense_3_loss: 0.4456 - val_loss: 0.8299 - val_dense_2_loss: 0.3707 - val_dense_3_loss: 0.4591
Epoch 67/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8013 - dense_2_loss: 0.3553 - dense_3_loss: 0.4458 - val_loss: 0.8289 - val_dense_2_loss: 0.3691 - val_dense_3_loss: 0.4597
Epoch 68/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7985 - dense_2_loss: 0.3548 - dense_3_loss: 0.4436 - val_loss: 0.8209 - val_dense_2_loss: 0.3677 - val_dense_3_loss: 0.4532
Epoch 69/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7968 - dense_2_loss: 0.3538 - dense_3_loss: 0.4428 - val_loss: 0.8211 - val_dense_2_loss: 0.3682 - val_dense_3_loss: 0.4527
Epoch 70/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7889 - dense_2_loss: 0.3510 - dense_3_loss: 0.4378 - val_loss: 0.8174 - val_dense_2_loss: 0.3667 - val_dense_3_loss: 0.4506
Epoch 71/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7877 - dense_2_loss: 0.3506 - dense_3_loss: 0.4373 - val_loss: 0.8115 - val_dense_2_loss: 0.3632 - val_dense_3_loss: 0.4482
Epoch 72/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7845 - dense_2_loss: 0.3496 - dense_3_loss: 0.4354 - val_loss: 0.8157 - val_dense_2_loss: 0.3665 - val_dense_3_loss: 0.4490
Epoch 73/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7818 - dense_2_loss: 0.3487 - dense_3_loss: 0.4332 - val_loss: 0.8084 - val_dense_2_loss: 0.3622 - val_dense_3_loss: 0.4461
Epoch 74/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.7843 - dense_2_loss: 0.3501 - dense_3_loss: 0.4348 - val_loss: 0.8084 - val_dense_2_loss: 0.3620 - val_dense_3_loss: 0.4463
Epoch 75/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7785 - dense_2_loss: 0.3474 - dense_3_loss: 0.4311 - val_loss: 0.8079 - val_dense_2_loss: 0.3622 - val_dense_3_loss: 0.4457
Epoch 76/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.7765 - dense_2_loss: 0.3470 - dense_3_loss: 0.4295 - val_loss: 0.8049 - val_dense_2_loss: 0.3639 - val_dense_3_loss: 0.4409
Epoch 77/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7735 - dense_2_loss: 0.3460 - dense_3_loss: 0.4276 - val_loss: 0.8012 - val_dense_2_loss: 0.3612 - val_dense_3_loss: 0.4399
Epoch 78/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7730 - dense_2_loss: 0.3459 - dense_3_loss: 0.4273 - val_loss: 0.8023 - val_dense_2_loss: 0.3603 - val_dense_3_loss: 0.4419
Epoch 79/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7756 - dense_2_loss: 0.3472 - dense_3_loss: 0.4284 - val_loss: 0.7986 - val_dense_2_loss: 0.3576 - val_dense_3_loss: 0.4409
Epoch 80/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7761 - dense_2_loss: 0.3475 - dense_3_loss: 0.4286 - val_loss: 0.7979 - val_dense_2_loss: 0.3598 - val_dense_3_loss: 0.4380
Epoch 81/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7733 - dense_2_loss: 0.3466 - dense_3_loss: 0.4267 - val_loss: 0.7936 - val_dense_2_loss: 0.3593 - val_dense_3_loss: 0.4341
Epoch 82/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7655 - dense_2_loss: 0.3430 - dense_3_loss: 0.4224 - val_loss: 0.7945 - val_dense_2_loss: 0.3580 - val_dense_3_loss: 0.4363
Epoch 83/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7669 - dense_2_loss: 0.3442 - dense_3_loss: 0.4224 - val_loss: 0.7913 - val_dense_2_loss: 0.3572 - val_dense_3_loss: 0.4340
Epoch 84/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7617 - dense_2_loss: 0.3424 - dense_3_loss: 0.4194 - val_loss: 0.7889 - val_dense_2_loss: 0.3561 - val_dense_3_loss: 0.4326
Epoch 85/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7601 - dense_2_loss: 0.3414 - dense_3_loss: 0.4186 - val_loss: 0.7866 - val_dense_2_loss: 0.3560 - val_dense_3_loss: 0.4304
Epoch 86/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7578 - dense_2_loss: 0.3408 - dense_3_loss: 0.4169 - val_loss: 0.7873 - val_dense_2_loss: 0.3562 - val_dense_3_loss: 0.4309
Epoch 87/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.7631 - dense_2_loss: 0.3432 - dense_3_loss: 0.4198 - val_loss: 0.7842 - val_dense_2_loss: 0.3540 - val_dense_3_loss: 0.4302
Epoch 88/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7603 - dense_2_loss: 0.3422 - dense_3_loss: 0.4181 - val_loss: 0.7832 - val_dense_2_loss: 0.3549 - val_dense_3_loss: 0.4282
Epoch 89/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7542 - dense_2_loss: 0.3391 - dense_3_loss: 0.4148 - val_loss: 0.7893 - val_dense_2_loss: 0.3571 - val_dense_3_loss: 0.4320
Epoch 90/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7528 - dense_2_loss: 0.3391 - dense_3_loss: 0.4139 - val_loss: 0.7808 - val_dense_2_loss: 0.3533 - val_dense_3_loss: 0.4274
Epoch 91/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7523 - dense_2_loss: 0.3385 - dense_3_loss: 0.4137 - val_loss: 0.7778 - val_dense_2_loss: 0.3532 - val_dense_3_loss: 0.4245
Epoch 92/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7514 - dense_2_loss: 0.3390 - dense_3_loss: 0.4130 - val_loss: 0.7748 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4241
Epoch 93/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.7479 - dense_2_loss: 0.3371 - dense_3_loss: 0.4106 - val_loss: 0.7771 - val_dense_2_loss: 0.3529 - val_dense_3_loss: 0.4241
Epoch 94/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7477 - dense_2_loss: 0.3371 - dense_3_loss: 0.4105 - val_loss: 0.7711 - val_dense_2_loss: 0.3500 - val_dense_3_loss: 0.4210
Epoch 95/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7519 - dense_2_loss: 0.3391 - dense_3_loss: 0.4126 - val_loss: 0.7720 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4212
Epoch 96/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7544 - dense_2_loss: 0.3404 - dense_3_loss: 0.4140 - val_loss: 0.7853 - val_dense_2_loss: 0.3559 - val_dense_3_loss: 0.4293
Epoch 97/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.7543 - dense_2_loss: 0.3404 - dense_3_loss: 0.4140 - val_loss: 0.7712 - val_dense_2_loss: 0.3506 - val_dense_3_loss: 0.4205
Epoch 98/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.7540 - dense_2_loss: 0.3405 - dense_3_loss: 0.4140 - val_loss: 0.7728 - val_dense_2_loss: 0.3520 - val_dense_3_loss: 0.4207
Epoch 99/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.7424 - dense_2_loss: 0.3352 - dense_3_loss: 0.4070 - val_loss: 0.7710 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4203
5160/1 [====================================================] - 0s 29us/sample - loss: 0.8831 - dense_2_loss: 0.3513 - dense_3_loss: 0.4160
[0.7661929805149403, 0.35130394, 0.41597202]

这里会有三组线，一个是总的，一个是第一个输出，一个是第二个输出。

基础API

我们现在来看一下tensorflow中最基本的常量操作

import tensorflow as tf
import numpy as np

if __name__ == "__main__":

    # 定义矩阵常量
    t = tf.constant([[1., 2., 3.], [4., 5., 6.]])
    print(t)
    print(t[:, 1:])
    print(t[:, 1])
    # 算子
    print(t + 10)
    # 计算平方
    print(tf.square(t))
    # 计算它本身和它转置的点乘
    print(t @ tf.transpose(t))
    # 获取numpy的值
    print(t.numpy())
    # 作为numpy的输入
    print(np.square(t))
    # 将numpy对象转成tensorflow对象
    np_t = np.array([[1., 2., 3.], [4., 5., 6.]])
    print(tf.constant(np_t))
    # 标量
    num = tf.constant(2.718)
    print(num)
    print(num.numpy())
    print(num.shape)
    # 字符串
    t = tf.constant("coffee")
    print(t)
    print(tf.strings.length(t))
    print(tf.strings.length(t, unit='UTF8_CHAR'))
    # 解码成utf8编码
    print(tf.strings.unicode_decode(t, 'UTF8'))
    # 字符串数组
    t = tf.constant(['cafe', 'coffee', '咖啡'])
    print(t)
    print(tf.strings.length(t, unit='UTF8_CHAR'))
    print(tf.strings.unicode_decode(t, 'UTF8'))
    # ragged tensor不等长的数组
    r = tf.ragged.constant([[11, 12], [21, 22, 23], [], [41]])
    print(r)
    print(r[1])
    print(r[1:3])
    # ragged tensor算子
    r2 = tf.ragged.constant([[51, 52], [], [71], [42, 43]])
    # 在第0个维度上进行拼接
    print(tf.concat([r, r2], axis=0))
    # 在第1个维度上进行拼接,如果第一个维度的数量不相等，会报错
    print(tf.concat([r, r2], axis=1))
    # 将ragged tensor转化为普通tensor，空缺位置补0
    print(r.to_tensor())
    # sparse tensor 大部分位置的值为0，只有少部分不为0
    # indices必须排序放置，否则转普通tensor会报错
    s = tf.SparseTensor(indices=[[0, 1], [1, 0], [2, 3]],
                        values=[1., 2., 3], dense_shape=[3, 4])
    print(s)
    # 将sparse tensor转化为普通的tensor
    print(tf.sparse.to_dense(s))
    # sparse tensor算子
    s2 = s * 2.0
    print(s2)
    try:
        s3 = s + 1.
    except TypeError as ex:
        print(ex)
    s4 = tf.constant([[10., 20.], [30., 40.], [50., 60.], [70., 80.]])
    # 矩阵乘法
    print(tf.sparse.sparse_dense_matmul(s, s4))
    s5 = tf.SparseTensor(indices=[[0, 2], [0, 1], [2, 3]],
                         values=[1., 2., 3], dense_shape=[3, 4])
    print(s5)
    try:
        print(tf.sparse.to_dense(s5))
    except Exception as ex:
        print(ex)
    s6 = tf.sparse.reorder(s5)
    print(tf.sparse.to_dense(s6))

运行结果

tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[2. 3.]
 [5. 6.]], shape=(2, 2), dtype=float32)
tf.Tensor([2. 5.], shape=(2,), dtype=float32)
tf.Tensor(
[[11. 12. 13.]
 [14. 15. 16.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[ 1.  4.  9.]
 [16. 25. 36.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[14. 32.]
 [32. 77.]], shape=(2, 2), dtype=float32)
[[1. 2. 3.]
 [4. 5. 6.]]
[[ 1.  4.  9.]
 [16. 25. 36.]]
tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float64)
tf.Tensor(2.718, shape=(), dtype=float32)
2.718
()
tf.Tensor(b'coffee', shape=(), dtype=string)
tf.Tensor(6, shape=(), dtype=int32)
tf.Tensor(6, shape=(), dtype=int32)
tf.Tensor([ 99 111 102 102 101 101], shape=(6,), dtype=int32)
tf.Tensor([b'cafe' b'coffee' b'\xe5\x92\x96\xe5\x95\xa1'], shape=(3,), dtype=string)
tf.Tensor([4 6 2], shape=(3,), dtype=int32)
<tf.RaggedTensor [[99, 97, 102, 101], [99, 111, 102, 102, 101, 101], [21654, 21857]]>
<tf.RaggedTensor [[11, 12], [21, 22, 23], [], [41]]>
tf.Tensor([21 22 23], shape=(3,), dtype=int32)
<tf.RaggedTensor [[21, 22, 23], []]>
<tf.RaggedTensor [[11, 12], [21, 22, 23], [], [41], [51, 52], [], [71], [42, 43]]>
<tf.RaggedTensor [[11, 12, 51, 52], [21, 22, 23], [71], [41, 42, 43]]>
tf.Tensor(
[[11 12  0]
 [21 22 23]
 [ 0  0  0]
 [41  0  0]], shape=(4, 3), dtype=int32)
SparseTensor(indices=tf.Tensor(
[[0 1]
 [1 0]
 [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))
tf.Tensor(
[[0. 1. 0. 0.]
 [2. 0. 0. 0.]
 [0. 0. 0. 3.]], shape=(3, 4), dtype=float32)
SparseTensor(indices=tf.Tensor(
[[0 1]
 [1 0]
 [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([2. 4. 6.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))
unsupported operand type(s) for +: 'SparseTensor' and 'float'
tf.Tensor(
[[ 30.  40.]
 [ 20.  40.]
 [210. 240.]], shape=(3, 2), dtype=float32)
SparseTensor(indices=tf.Tensor(
[[0 2]
 [0 1]
 [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))
indices[1] = [0,1] is out of order [Op:SparseToDense]
tf.Tensor(
[[0. 2. 1. 0.]
 [0. 0. 0. 0.]
 [0. 0. 0. 3.]], shape=(3, 4), dtype=float32)

现在我们来看一下它的变量操作

import tensorflow as tf
import numpy as np

if __name__ == "__main__":

    v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])
    print(v)
    # 转化为常量
    print(v.value())
    print(v.numpy())
    # 重新赋值
    v.assign(2 * v)
    print(v)
    # 给变量的某个位置重新赋值
    v[0, 1].assign(42)
    print(v)
    # 给变量的某一行重新赋值
    v[1].assign([7., 8., 9.])
    print(v)
    # 不能直接使用=赋值
    try:
        v[0] = [10., 20., 30.]
    except TypeError as ex:
        print(ex)

运行结果

<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[1., 2., 3.],
       [4., 5., 6.]], dtype=float32)>
tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float32)
[[1. 2. 3.]
 [4. 5. 6.]]
<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[ 2.,  4.,  6.],
       [ 8., 10., 12.]], dtype=float32)>
<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[ 2., 42.,  6.],
       [ 8., 10., 12.]], dtype=float32)>
<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=
array([[ 2., 42.,  6.],
       [ 7.,  8.,  9.]], dtype=float32)>
'ResourceVariable' object does not support item assignment

自定义损失函数

这里我们还是以加利福尼亚的房价回归问题来说明

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 归一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)

    def customized_mse(y_true, y_pred):
        # 自定义损失函数
        return tf.reduce_mean(tf.square(y_pred - y_true))

    model = keras.models.Sequential([
        layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
        layers.Dense(1)
    ])
    print(model.summary())
    # 使用均方误差损失函数和随机梯度下降法
    model.compile(loss=customized_mse, optimizer=optimizers.SGD(learning_rate=0.01),
                  metrics=['mean_squared_error'])
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 学习曲线
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, 30)                270       
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 31        
=================================================================
Total params: 301
Trainable params: 301
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 119us/sample - loss: 1.5018 - mean_squared_error: 1.5018 - val_loss: 0.8721 - val_mean_squared_error: 0.8721
Epoch 2/100
11610/11610 [==============================] - 0s 43us/sample - loss: 1.8099 - mean_squared_error: 1.8099 - val_loss: 0.5967 - val_mean_squared_error: 0.5967
Epoch 3/100
11610/11610 [==============================] - 1s 43us/sample - loss: 0.4971 - mean_squared_error: 0.4971 - val_loss: 0.4621 - val_mean_squared_error: 0.4621
Epoch 4/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.4396 - mean_squared_error: 0.4396 - val_loss: 0.4405 - val_mean_squared_error: 0.4405
Epoch 5/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.4121 - mean_squared_error: 0.4121 - val_loss: 0.4133 - val_mean_squared_error: 0.4133
Epoch 6/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.4043 - mean_squared_error: 0.4043 - val_loss: 0.4039 - val_mean_squared_error: 0.4039
Epoch 7/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3896 - mean_squared_error: 0.3896 - val_loss: 0.4000 - val_mean_squared_error: 0.4000
Epoch 8/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3796 - mean_squared_error: 0.3796 - val_loss: 0.3936 - val_mean_squared_error: 0.3936
Epoch 9/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3771 - mean_squared_error: 0.3771 - val_loss: 0.5541 - val_mean_squared_error: 0.5541
Epoch 10/100
11610/11610 [==============================] - 1s 46us/sample - loss: 0.3820 - mean_squared_error: 0.3820 - val_loss: 0.3874 - val_mean_squared_error: 0.3874
Epoch 11/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3686 - mean_squared_error: 0.3686 - val_loss: 0.3758 - val_mean_squared_error: 0.3758
Epoch 12/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3666 - mean_squared_error: 0.3666 - val_loss: 0.3867 - val_mean_squared_error: 0.3867
Epoch 13/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3632 - mean_squared_error: 0.3632 - val_loss: 0.3711 - val_mean_squared_error: 0.3711
Epoch 14/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3592 - mean_squared_error: 0.3592 - val_loss: 0.3790 - val_mean_squared_error: 0.3790
Epoch 15/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3578 - mean_squared_error: 0.3578 - val_loss: 0.3687 - val_mean_squared_error: 0.3687
Epoch 16/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3547 - mean_squared_error: 0.3547 - val_loss: 0.3691 - val_mean_squared_error: 0.3691
Epoch 17/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3530 - mean_squared_error: 0.3530 - val_loss: 0.3690 - val_mean_squared_error: 0.3690
Epoch 18/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.3549 - mean_squared_error: 0.3549 - val_loss: 0.3625 - val_mean_squared_error: 0.3625
Epoch 19/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.3506 - mean_squared_error: 0.3506 - val_loss: 0.3605 - val_mean_squared_error: 0.3605
Epoch 20/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3481 - mean_squared_error: 0.3481 - val_loss: 0.3610 - val_mean_squared_error: 0.3610
Epoch 21/100
11610/11610 [==============================] - 1s 72us/sample - loss: 0.3479 - mean_squared_error: 0.3479 - val_loss: 0.3601 - val_mean_squared_error: 0.3601
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3448 - mean_squared_error: 0.3448 - val_loss: 0.3578 - val_mean_squared_error: 0.3578
Epoch 23/100
11610/11610 [==============================] - 1s 73us/sample - loss: 0.3457 - mean_squared_error: 0.3457 - val_loss: 0.3611 - val_mean_squared_error: 0.3611
Epoch 24/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3447 - mean_squared_error: 0.3447 - val_loss: 0.3539 - val_mean_squared_error: 0.3539
Epoch 25/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3432 - mean_squared_error: 0.3432 - val_loss: 0.3515 - val_mean_squared_error: 0.3515
Epoch 26/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3404 - mean_squared_error: 0.3404 - val_loss: 0.3544 - val_mean_squared_error: 0.3544
Epoch 27/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3406 - mean_squared_error: 0.3406 - val_loss: 0.3504 - val_mean_squared_error: 0.3504
Epoch 28/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3389 - mean_squared_error: 0.3389 - val_loss: 0.3514 - val_mean_squared_error: 0.3514
Epoch 29/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3385 - mean_squared_error: 0.3385 - val_loss: 0.3470 - val_mean_squared_error: 0.3470
Epoch 30/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3375 - mean_squared_error: 0.3375 - val_loss: 0.3513 - val_mean_squared_error: 0.3513
Epoch 31/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3362 - mean_squared_error: 0.3362 - val_loss: 0.3474 - val_mean_squared_error: 0.3474
Epoch 32/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3351 - mean_squared_error: 0.3351 - val_loss: 0.3493 - val_mean_squared_error: 0.3493
Epoch 33/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3349 - mean_squared_error: 0.3349 - val_loss: 0.3461 - val_mean_squared_error: 0.3461
Epoch 34/100
11610/11610 [==============================] - 0s 41us/sample - loss: 0.3326 - mean_squared_error: 0.3326 - val_loss: 0.3464 - val_mean_squared_error: 0.3464
5160/1 [=====================================================] - 0s 20us/sample - loss: 0.5104 - mean_squared_error: 0.3525
[0.3524518048116403, 0.3524517]

这里我们可以看到我们自己实现的均方误差损失函数和系统自带的均方误差损失函数是一样，他们损失函数值曲线是重合的。

自定义Layers

我们先来看一下Layers的一些特性

import tensorflow as tf
from tensorflow.keras import layers

if __name__ == "__main__":

    # 我们一般会在全连接层的第一层的时候输入input_shape
    # 不使用input_shape也没关系，它会在第一次运行的时候推导出来
    layer = layers.Dense(100, input_shape=(None, 5))
    # 这里会点乘一个[5, 100]的矩阵转成[None, 100]的输出
    print(layer(tf.zeros([10, 5])))
    # 打印该层所有的参数
    print(layer.variables)
    # 打印该层所有可训练的参数
    print(layer.trainable_variables)

运行结果

tf.Tensor(
[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0.]], shape=(10, 100), dtype=float32)
[<tf.Variable 'dense/kernel:0' shape=(5, 100) dtype=float32, numpy=
array([[ 0.17777802, -0.0925342 ,  0.22044905, -0.1196067 , -0.15553904,
        -0.19373423, -0.13884991, -0.23252396, -0.01252402, -0.09044193,
        -0.06529157,  0.22907178, -0.1411261 ,  0.20145597,  0.11501376,
        -0.15268266, -0.17305377, -0.01125969, -0.10907765,  0.0403709 ,
         0.06923743, -0.00155647, -0.18127626,  0.11550726,  0.03372289,
         0.13893865, -0.0092984 , -0.20496821, -0.02627921,  0.01125975,
        -0.04647307,  0.15463649,  0.17841218,  0.17077957, -0.0842554 ,
         0.0047874 , -0.12141771,  0.22519882,  0.23548423,  0.06460319,
        -0.03109986, -0.12173892, -0.02452356, -0.04257253,  0.2064407 ,
        -0.06991957,  0.06425266, -0.1007075 ,  0.19131865, -0.1955776 ,
         0.21193884, -0.2004574 , -0.07692617, -0.08107474,  0.1934263 ,
         0.05712549,  0.01886164, -0.12672797, -0.17843167, -0.05964956,
        -0.238883  , -0.21325174,  0.11310308, -0.00318795, -0.12962502,
        -0.16521364,  0.12938736, -0.23181748,  0.21444045, -0.12927116,
        -0.18740693,  0.05530755,  0.0146782 ,  0.21035798, -0.15627733,
        -0.01847358, -0.08605188, -0.0206164 ,  0.20378043, -0.03834918,
         0.0639727 , -0.06449054,  0.17925756, -0.14535832,  0.22247057,
         0.08074225,  0.19761471, -0.20858642,  0.17465816, -0.12256925,
        -0.07192959, -0.01034951, -0.2167171 ,  0.02262951, -0.2061531 ,
         0.19094323, -0.0974756 ,  0.21508281, -0.01393197,  0.19384028],
       [-0.02444541, -0.21043453,  0.06121971, -0.18661837, -0.1679871 ,
         0.17915292,  0.0956945 , -0.03815825, -0.10663904, -0.04658341,
        -0.18815587, -0.04888278,  0.19361894, -0.13932027, -0.2013808 ,
        -0.2031331 , -0.12357853,  0.15468134, -0.16377276, -0.15836287,
        -0.03290932,  0.09049819, -0.10049133, -0.1802709 ,  0.11397789,
        -0.22433408, -0.01369557, -0.21804577,  0.13998653, -0.0452639 ,
        -0.10171422, -0.15374628, -0.07929   ,  0.08984323,  0.11814152,
         0.13589476,  0.05618854,  0.1663635 , -0.01764336,  0.09068643,
        -0.1157929 , -0.1266903 , -0.07107972,  0.15617536, -0.1517889 ,
        -0.21622056, -0.17632967,  0.14407007,  0.10903122, -0.00500569,
         0.05290176, -0.03641239, -0.2377142 , -0.12242448, -0.07954802,
         0.09190597, -0.09879458, -0.18804912, -0.14522013,  0.07479332,
         0.17024298,  0.0161783 , -0.0524762 ,  0.0176069 , -0.08270177,
         0.16904478,  0.17900594,  0.15703659,  0.19818316, -0.1625318 ,
        -0.12794688, -0.1563586 , -0.09517953,  0.09158082,  0.23767962,
         0.20551957,  0.14481695,  0.2045602 , -0.06479037,  0.05358954,
         0.1212578 , -0.02165481, -0.1284295 , -0.08271579, -0.05446866,
         0.05413379,  0.00234202,  0.22771172, -0.2224471 , -0.03084002,
        -0.19375737,  0.0798779 , -0.09207922,  0.13679741, -0.1429534 ,
         0.0209298 , -0.0841022 , -0.16638935,  0.0903822 , -0.19967408],
       [ 0.11377819, -0.03955515,  0.02814175,  0.04407446,  0.02015819,
        -0.01894131,  0.14453827,  0.07663743, -0.2206468 , -0.23865755,
        -0.23881991,  0.01236854, -0.09480104,  0.20786951, -0.02678156,
        -0.0305012 , -0.05566147, -0.21860714, -0.1938608 ,  0.053019  ,
        -0.05170445, -0.1206295 , -0.22782467,  0.06737016,  0.05737154,
         0.01793937, -0.2030854 , -0.14422959, -0.17374498, -0.02698712,
        -0.13331282,  0.12249409, -0.19247851,  0.14200823, -0.16609323,
         0.14399786,  0.21913432, -0.21645993, -0.191912  ,  0.0571617 ,
         0.19024946,  0.08669983,  0.08364411, -0.1892563 , -0.02523425,
         0.11180784,  0.00898317,  0.03620596, -0.08056329,  0.19991381,
         0.11432077,  0.1168678 , -0.1644195 ,  0.00769468,  0.22227935,
        -0.20849067,  0.15359052,  0.1345963 , -0.21721914,  0.0614645 ,
         0.12685539,  0.11890687, -0.20878732, -0.15853551,  0.0458544 ,
        -0.15758611, -0.16187118, -0.09367229, -0.16155185, -0.18485507,
        -0.13647649, -0.12254263, -0.05565156, -0.18738703, -0.06917199,
        -0.03574945,  0.17475249, -0.17476122, -0.10158296,  0.12375195,
         0.18355547,  0.03811152,  0.07888819,  0.09909324, -0.15394625,
         0.11169995, -0.09254411, -0.10357954,  0.19252472,  0.02573626,
        -0.01318276, -0.09475978, -0.1874548 , -0.19183636, -0.18057513,
        -0.15745679,  0.18442439,  0.08096658, -0.01627833,  0.05583473],
       [-0.04097484,  0.23634328, -0.1889084 ,  0.08590437,  0.17894466,
        -0.11124423,  0.18041344, -0.21274638,  0.11614217,  0.10249467,
         0.21891718,  0.19230996,  0.15266438, -0.10032661, -0.22356763,
         0.11601089, -0.08055103,  0.00928256,  0.16465248, -0.04512097,
        -0.17905587, -0.23013544, -0.00562099, -0.09802979, -0.22574061,
         0.02589588,  0.19565006,  0.09272386, -0.13209037, -0.21364078,
        -0.15901634,  0.21333267, -0.22375406, -0.17148429,  0.15592472,
        -0.1618546 ,  0.20303024,  0.04697676,  0.20634227, -0.1878765 ,
         0.22583528, -0.12808219,  0.18875875, -0.19559893,  0.15292238,
         0.08833222,  0.1192212 ,  0.12659721,  0.12049772, -0.21185097,
         0.16098978, -0.11107406, -0.13046448,  0.21239029,  0.00350848,
         0.15068768, -0.0189786 ,  0.1369033 , -0.10414104,  0.16339312,
        -0.09338254, -0.14816928, -0.2035834 ,  0.16379024,  0.01705651,
         0.11921181,  0.23129468,  0.18642814, -0.17961605,  0.22902219,
         0.2109852 , -0.19536257,  0.06335585,  0.05963777,  0.0976281 ,
         0.00718886,  0.02180146,  0.16579719, -0.23447831, -0.22135682,
        -0.15220946, -0.03425349, -0.16059369, -0.17343065,  0.12119611,
         0.07790975, -0.07582501, -0.03742242,  0.18471824,  0.20959799,
        -0.22154935, -0.2013107 , -0.12604958,  0.12774341, -0.01246983,
        -0.18907636,  0.03709231, -0.02248657,  0.03133444, -0.1615734 ],
       [-0.16601531,  0.21792419, -0.1103511 ,  0.0025705 , -0.05631484,
         0.07372896,  0.18572135, -0.07337801, -0.01120584, -0.13398278,
        -0.04314138,  0.17321984,  0.11909981, -0.15493754,  0.07232957,
         0.13085563, -0.2287109 , -0.21197955,  0.1428767 ,  0.1137528 ,
        -0.14559013,  0.13985835,  0.12416814, -0.05637537, -0.0046737 ,
         0.00876671,  0.05863507, -0.03590567,  0.10624601, -0.1412015 ,
         0.15561827,  0.10837926, -0.1810194 , -0.05347614,  0.03898682,
        -0.10682511, -0.16920623, -0.18795235, -0.08359696,  0.18185644,
        -0.19991723, -0.05360089,  0.13864844,  0.1414309 , -0.23352294,
         0.21933769, -0.2087497 ,  0.02414815, -0.20864718, -0.01974651,
        -0.12589416, -0.04501775,  0.09451391,  0.01687138,  0.13819186,
        -0.21920864,  0.22943564,  0.22025959,  0.07460229, -0.11851113,
         0.04449643, -0.11150999,  0.18484862,  0.18422444, -0.19684097,
         0.125441  ,  0.16443445,  0.174614  ,  0.21528427, -0.02458197,
         0.2311335 , -0.20051023,  0.16148575,  0.03468569,  0.08386306,
         0.18408652, -0.09470819, -0.22607385, -0.04776055,  0.17687435,
         0.01153742,  0.15740515,  0.22579281,  0.2256033 ,  0.23760812,
        -0.08624862, -0.04818098, -0.09573658,  0.00707476, -0.174594  ,
         0.17612858, -0.02651523, -0.03352439, -0.0982894 ,  0.12405153,
         0.1423658 ,  0.12438641, -0.22709328,  0.18641399, -0.0159114 ]],
      dtype=float32)>, <tf.Variable 'dense/bias:0' shape=(100,) dtype=float32, numpy=
array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
      dtype=float32)>]
[<tf.Variable 'dense/kernel:0' shape=(5, 100) dtype=float32, numpy=
array([[ 0.17777802, -0.0925342 ,  0.22044905, -0.1196067 , -0.15553904,
        -0.19373423, -0.13884991, -0.23252396, -0.01252402, -0.09044193,
        -0.06529157,  0.22907178, -0.1411261 ,  0.20145597,  0.11501376,
        -0.15268266, -0.17305377, -0.01125969, -0.10907765,  0.0403709 ,
         0.06923743, -0.00155647, -0.18127626,  0.11550726,  0.03372289,
         0.13893865, -0.0092984 , -0.20496821, -0.02627921,  0.01125975,
        -0.04647307,  0.15463649,  0.17841218,  0.17077957, -0.0842554 ,
         0.0047874 , -0.12141771,  0.22519882,  0.23548423,  0.06460319,
        -0.03109986, -0.12173892, -0.02452356, -0.04257253,  0.2064407 ,
        -0.06991957,  0.06425266, -0.1007075 ,  0.19131865, -0.1955776 ,
         0.21193884, -0.2004574 , -0.07692617, -0.08107474,  0.1934263 ,
         0.05712549,  0.01886164, -0.12672797, -0.17843167, -0.05964956,
        -0.238883  , -0.21325174,  0.11310308, -0.00318795, -0.12962502,
        -0.16521364,  0.12938736, -0.23181748,  0.21444045, -0.12927116,
        -0.18740693,  0.05530755,  0.0146782 ,  0.21035798, -0.15627733,
        -0.01847358, -0.08605188, -0.0206164 ,  0.20378043, -0.03834918,
         0.0639727 , -0.06449054,  0.17925756, -0.14535832,  0.22247057,
         0.08074225,  0.19761471, -0.20858642,  0.17465816, -0.12256925,
        -0.07192959, -0.01034951, -0.2167171 ,  0.02262951, -0.2061531 ,
         0.19094323, -0.0974756 ,  0.21508281, -0.01393197,  0.19384028],
       [-0.02444541, -0.21043453,  0.06121971, -0.18661837, -0.1679871 ,
         0.17915292,  0.0956945 , -0.03815825, -0.10663904, -0.04658341,
        -0.18815587, -0.04888278,  0.19361894, -0.13932027, -0.2013808 ,
        -0.2031331 , -0.12357853,  0.15468134, -0.16377276, -0.15836287,
        -0.03290932,  0.09049819, -0.10049133, -0.1802709 ,  0.11397789,
        -0.22433408, -0.01369557, -0.21804577,  0.13998653, -0.0452639 ,
        -0.10171422, -0.15374628, -0.07929   ,  0.08984323,  0.11814152,
         0.13589476,  0.05618854,  0.1663635 , -0.01764336,  0.09068643,
        -0.1157929 , -0.1266903 , -0.07107972,  0.15617536, -0.1517889 ,
        -0.21622056, -0.17632967,  0.14407007,  0.10903122, -0.00500569,
         0.05290176, -0.03641239, -0.2377142 , -0.12242448, -0.07954802,
         0.09190597, -0.09879458, -0.18804912, -0.14522013,  0.07479332,
         0.17024298,  0.0161783 , -0.0524762 ,  0.0176069 , -0.08270177,
         0.16904478,  0.17900594,  0.15703659,  0.19818316, -0.1625318 ,
        -0.12794688, -0.1563586 , -0.09517953,  0.09158082,  0.23767962,
         0.20551957,  0.14481695,  0.2045602 , -0.06479037,  0.05358954,
         0.1212578 , -0.02165481, -0.1284295 , -0.08271579, -0.05446866,
         0.05413379,  0.00234202,  0.22771172, -0.2224471 , -0.03084002,
        -0.19375737,  0.0798779 , -0.09207922,  0.13679741, -0.1429534 ,
         0.0209298 , -0.0841022 , -0.16638935,  0.0903822 , -0.19967408],
       [ 0.11377819, -0.03955515,  0.02814175,  0.04407446,  0.02015819,
        -0.01894131,  0.14453827,  0.07663743, -0.2206468 , -0.23865755,
        -0.23881991,  0.01236854, -0.09480104,  0.20786951, -0.02678156,
        -0.0305012 , -0.05566147, -0.21860714, -0.1938608 ,  0.053019  ,
        -0.05170445, -0.1206295 , -0.22782467,  0.06737016,  0.05737154,
         0.01793937, -0.2030854 , -0.14422959, -0.17374498, -0.02698712,
        -0.13331282,  0.12249409, -0.19247851,  0.14200823, -0.16609323,
         0.14399786,  0.21913432, -0.21645993, -0.191912  ,  0.0571617 ,
         0.19024946,  0.08669983,  0.08364411, -0.1892563 , -0.02523425,
         0.11180784,  0.00898317,  0.03620596, -0.08056329,  0.19991381,
         0.11432077,  0.1168678 , -0.1644195 ,  0.00769468,  0.22227935,
        -0.20849067,  0.15359052,  0.1345963 , -0.21721914,  0.0614645 ,
         0.12685539,  0.11890687, -0.20878732, -0.15853551,  0.0458544 ,
        -0.15758611, -0.16187118, -0.09367229, -0.16155185, -0.18485507,
        -0.13647649, -0.12254263, -0.05565156, -0.18738703, -0.06917199,
        -0.03574945,  0.17475249, -0.17476122, -0.10158296,  0.12375195,
         0.18355547,  0.03811152,  0.07888819,  0.09909324, -0.15394625,
         0.11169995, -0.09254411, -0.10357954,  0.19252472,  0.02573626,
        -0.01318276, -0.09475978, -0.1874548 , -0.19183636, -0.18057513,
        -0.15745679,  0.18442439,  0.08096658, -0.01627833,  0.05583473],
       [-0.04097484,  0.23634328, -0.1889084 ,  0.08590437,  0.17894466,
        -0.11124423,  0.18041344, -0.21274638,  0.11614217,  0.10249467,
         0.21891718,  0.19230996,  0.15266438, -0.10032661, -0.22356763,
         0.11601089, -0.08055103,  0.00928256,  0.16465248, -0.04512097,
        -0.17905587, -0.23013544, -0.00562099, -0.09802979, -0.22574061,
         0.02589588,  0.19565006,  0.09272386, -0.13209037, -0.21364078,
        -0.15901634,  0.21333267, -0.22375406, -0.17148429,  0.15592472,
        -0.1618546 ,  0.20303024,  0.04697676,  0.20634227, -0.1878765 ,
         0.22583528, -0.12808219,  0.18875875, -0.19559893,  0.15292238,
         0.08833222,  0.1192212 ,  0.12659721,  0.12049772, -0.21185097,
         0.16098978, -0.11107406, -0.13046448,  0.21239029,  0.00350848,
         0.15068768, -0.0189786 ,  0.1369033 , -0.10414104,  0.16339312,
        -0.09338254, -0.14816928, -0.2035834 ,  0.16379024,  0.01705651,
         0.11921181,  0.23129468,  0.18642814, -0.17961605,  0.22902219,
         0.2109852 , -0.19536257,  0.06335585,  0.05963777,  0.0976281 ,
         0.00718886,  0.02180146,  0.16579719, -0.23447831, -0.22135682,
        -0.15220946, -0.03425349, -0.16059369, -0.17343065,  0.12119611,
         0.07790975, -0.07582501, -0.03742242,  0.18471824,  0.20959799,
        -0.22154935, -0.2013107 , -0.12604958,  0.12774341, -0.01246983,
        -0.18907636,  0.03709231, -0.02248657,  0.03133444, -0.1615734 ],
       [-0.16601531,  0.21792419, -0.1103511 ,  0.0025705 , -0.05631484,
         0.07372896,  0.18572135, -0.07337801, -0.01120584, -0.13398278,
        -0.04314138,  0.17321984,  0.11909981, -0.15493754,  0.07232957,
         0.13085563, -0.2287109 , -0.21197955,  0.1428767 ,  0.1137528 ,
        -0.14559013,  0.13985835,  0.12416814, -0.05637537, -0.0046737 ,
         0.00876671,  0.05863507, -0.03590567,  0.10624601, -0.1412015 ,
         0.15561827,  0.10837926, -0.1810194 , -0.05347614,  0.03898682,
        -0.10682511, -0.16920623, -0.18795235, -0.08359696,  0.18185644,
        -0.19991723, -0.05360089,  0.13864844,  0.1414309 , -0.23352294,
         0.21933769, -0.2087497 ,  0.02414815, -0.20864718, -0.01974651,
        -0.12589416, -0.04501775,  0.09451391,  0.01687138,  0.13819186,
        -0.21920864,  0.22943564,  0.22025959,  0.07460229, -0.11851113,
         0.04449643, -0.11150999,  0.18484862,  0.18422444, -0.19684097,
         0.125441  ,  0.16443445,  0.174614  ,  0.21528427, -0.02458197,
         0.2311335 , -0.20051023,  0.16148575,  0.03468569,  0.08386306,
         0.18408652, -0.09470819, -0.22607385, -0.04776055,  0.17687435,
         0.01153742,  0.15740515,  0.22579281,  0.2256033 ,  0.23760812,
        -0.08624862, -0.04818098, -0.09573658,  0.00707476, -0.174594  ,
         0.17612858, -0.02651523, -0.03352439, -0.0982894 ,  0.12405153,
         0.1423658 ,  0.12438641, -0.22709328,  0.18641399, -0.0159114 ]],
      dtype=float32)>, <tf.Variable 'dense/bias:0' shape=(100,) dtype=float32, numpy=
array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
      dtype=float32)>]

通过结果我们可以看出layer的输出结果是一个常量，而它的参数是两个变量。其中dense/kernel:0就是我们经常说的线性模型Wx+b中的系数W，而dense/bias:0就是其中的截距b。而且在这里它的参数和可训练的参数是一致的。现在我们来实现自定义的Dense_Layer。

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 归一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)

    def customized_mse(y_true, y_pred):
        # 自定义损失函数
        return tf.reduce_mean(tf.square(y_pred - y_true))

    class CustomizedDenseLayer(layers.Layer):
        # 自定义DenseLayer

        def __init__(self, units, activation=None, **kwargs):
            '''
            :param units: 输出大小
            :param activation: 激活函数
            :param kwargs:
            '''
            self.units = units
            self.activation = layers.Activation(activation=activation)
            super(CustomizedDenseLayer, self).__init__(**kwargs)

        def build(self, input_shape):
            '''
            构建所需要的参数
            :param input_shape: 输入的形状
            :return:
            '''
            # Wx + b，input_shape:[None, a],W:[a, b],output_shape:[None, b]
            # shape为W的形状，即变换矩阵
            # uniform表示均匀分布的方式初始化
            # trainable = True表示参数是可以被训练的
            self.kernel = self.add_weight(name='kernel',
                                          shape=(input_shape[1], self.units),
                                          initializer='uniform',
                                          trainable=True)
            self.bias = self.add_weight(name='bias',
                                        shape=(self.units,),
                                        initializer='zeros',
                                        trainable=True)
            super(CustomizedDenseLayer, self).build(input_shape)

        def call(self, X):
            # 完成正向计算
            return self.activation(X @ self.kernel + self.bias)

    model = keras.models.Sequential([
        CustomizedDenseLayer(30, activation='relu', input_shape=X_train.shape[1:]),
        CustomizedDenseLayer(1)
    ])
    print(model.summary())
    # 使用均方误差损失函数和随机梯度下降法
    model.compile(loss=customized_mse, optimizer=optimizers.SGD(learning_rate=0.01),
                  metrics=['mean_squared_error'])
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 学习曲线
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

运行结果

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
customized_dense_layer (Cust (None, 30)                270       
_________________________________________________________________
customized_dense_layer_1 (Cu (None, 1)                 31        
=================================================================
Total params: 301
Trainable params: 301
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 98us/sample - loss: 1.1060 - mean_squared_error: 1.1061 - val_loss: 0.6408 - val_mean_squared_error: 0.6408
Epoch 2/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.5543 - mean_squared_error: 0.5543 - val_loss: 0.5453 - val_mean_squared_error: 0.5453
Epoch 3/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4889 - mean_squared_error: 0.4889 - val_loss: 0.4974 - val_mean_squared_error: 0.4974
Epoch 4/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4619 - mean_squared_error: 0.4619 - val_loss: 0.4884 - val_mean_squared_error: 0.4884
Epoch 5/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4479 - mean_squared_error: 0.4479 - val_loss: 0.4610 - val_mean_squared_error: 0.4610
Epoch 6/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4337 - mean_squared_error: 0.4337 - val_loss: 0.4442 - val_mean_squared_error: 0.4442
Epoch 7/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4234 - mean_squared_error: 0.4234 - val_loss: 0.4362 - val_mean_squared_error: 0.4362
Epoch 8/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.4155 - mean_squared_error: 0.4155 - val_loss: 0.4295 - val_mean_squared_error: 0.4295
Epoch 9/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4164 - mean_squared_error: 0.4164 - val_loss: 0.4230 - val_mean_squared_error: 0.4230
Epoch 10/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4067 - mean_squared_error: 0.4067 - val_loss: 0.4183 - val_mean_squared_error: 0.4183
Epoch 11/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3986 - mean_squared_error: 0.3986 - val_loss: 0.4144 - val_mean_squared_error: 0.4144
Epoch 12/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3946 - mean_squared_error: 0.3946 - val_loss: 0.4088 - val_mean_squared_error: 0.4088
Epoch 13/100
11610/11610 [==============================] - 1s 72us/sample - loss: 0.3923 - mean_squared_error: 0.3923 - val_loss: 0.4127 - val_mean_squared_error: 0.4127
Epoch 14/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3895 - mean_squared_error: 0.3895 - val_loss: 0.4019 - val_mean_squared_error: 0.4019
Epoch 15/100
11610/11610 [==============================] - 1s 59us/sample - loss: 0.3863 - mean_squared_error: 0.3863 - val_loss: 0.4008 - val_mean_squared_error: 0.4008
Epoch 16/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3835 - mean_squared_error: 0.3835 - val_loss: 0.3949 - val_mean_squared_error: 0.3949
Epoch 17/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3819 - mean_squared_error: 0.3819 - val_loss: 0.3942 - val_mean_squared_error: 0.3942
Epoch 18/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3813 - mean_squared_error: 0.3813 - val_loss: 0.3909 - val_mean_squared_error: 0.3909
Epoch 19/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3784 - mean_squared_error: 0.3784 - val_loss: 0.3882 - val_mean_squared_error: 0.3882
Epoch 20/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3763 - mean_squared_error: 0.3763 - val_loss: 0.3960 - val_mean_squared_error: 0.3960
Epoch 21/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3760 - mean_squared_error: 0.3760 - val_loss: 0.3876 - val_mean_squared_error: 0.3876
Epoch 22/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3730 - mean_squared_error: 0.3730 - val_loss: 0.3885 - val_mean_squared_error: 0.3885
Epoch 23/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3739 - mean_squared_error: 0.3739 - val_loss: 0.3854 - val_mean_squared_error: 0.3854
Epoch 24/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3701 - mean_squared_error: 0.3701 - val_loss: 0.3812 - val_mean_squared_error: 0.3812
Epoch 25/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3697 - mean_squared_error: 0.3697 - val_loss: 0.3796 - val_mean_squared_error: 0.3796
Epoch 26/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3701 - mean_squared_error: 0.3701 - val_loss: 0.3787 - val_mean_squared_error: 0.3787
Epoch 27/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3670 - mean_squared_error: 0.3670 - val_loss: 0.3777 - val_mean_squared_error: 0.3777
Epoch 28/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3661 - mean_squared_error: 0.3661 - val_loss: 0.3761 - val_mean_squared_error: 0.3761
Epoch 29/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3638 - mean_squared_error: 0.3638 - val_loss: 0.3738 - val_mean_squared_error: 0.3738
Epoch 30/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3677 - mean_squared_error: 0.3677 - val_loss: 0.3726 - val_mean_squared_error: 0.3726
Epoch 31/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3637 - mean_squared_error: 0.3637 - val_loss: 0.3703 - val_mean_squared_error: 0.3703
Epoch 32/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3651 - mean_squared_error: 0.3651 - val_loss: 0.3750 - val_mean_squared_error: 0.3750
Epoch 33/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3627 - mean_squared_error: 0.3627 - val_loss: 0.3714 - val_mean_squared_error: 0.3714
Epoch 34/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3593 - mean_squared_error: 0.3593 - val_loss: 0.3701 - val_mean_squared_error: 0.3701
Epoch 35/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3582 - mean_squared_error: 0.3582 - val_loss: 0.3681 - val_mean_squared_error: 0.3681
Epoch 36/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3622 - mean_squared_error: 0.3622 - val_loss: 0.3668 - val_mean_squared_error: 0.3668
Epoch 37/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3556 - mean_squared_error: 0.3556 - val_loss: 0.3789 - val_mean_squared_error: 0.3789
Epoch 38/100
11610/11610 [==============================] - 1s 47us/sample - loss: 0.3618 - mean_squared_error: 0.3618 - val_loss: 0.3651 - val_mean_squared_error: 0.3651
Epoch 39/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.3546 - mean_squared_error: 0.3546 - val_loss: 0.3630 - val_mean_squared_error: 0.3630
Epoch 40/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3533 - mean_squared_error: 0.3533 - val_loss: 0.3897 - val_mean_squared_error: 0.3897
Epoch 41/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3589 - mean_squared_error: 0.3589 - val_loss: 0.3621 - val_mean_squared_error: 0.3621
Epoch 42/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.3540 - mean_squared_error: 0.3540 - val_loss: 0.3648 - val_mean_squared_error: 0.3648
Epoch 43/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.3554 - mean_squared_error: 0.3554 - val_loss: 0.3638 - val_mean_squared_error: 0.3638
Epoch 44/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3512 - mean_squared_error: 0.3512 - val_loss: 0.3575 - val_mean_squared_error: 0.3575
Epoch 45/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3549 - mean_squared_error: 0.3549 - val_loss: 0.3566 - val_mean_squared_error: 0.3566
Epoch 46/100
11610/11610 [==============================] - 1s 79us/sample - loss: 0.3497 - mean_squared_error: 0.3497 - val_loss: 0.3561 - val_mean_squared_error: 0.3561
Epoch 47/100
11610/11610 [==============================] - 1s 86us/sample - loss: 0.3470 - mean_squared_error: 0.3470 - val_loss: 0.3584 - val_mean_squared_error: 0.3584
Epoch 48/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3470 - mean_squared_error: 0.3470 - val_loss: 0.3526 - val_mean_squared_error: 0.3526
Epoch 49/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3469 - mean_squared_error: 0.3469 - val_loss: 0.3529 - val_mean_squared_error: 0.3529
Epoch 50/100
11610/11610 [==============================] - 1s 60us/sample - loss: 0.3492 - mean_squared_error: 0.3492 - val_loss: 0.3531 - val_mean_squared_error: 0.3531
Epoch 51/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3428 - mean_squared_error: 0.3428 - val_loss: 0.3601 - val_mean_squared_error: 0.3601
Epoch 52/100
11610/11610 [==============================] - 1s 73us/sample - loss: 0.3448 - mean_squared_error: 0.3448 - val_loss: 0.3517 - val_mean_squared_error: 0.3517
Epoch 53/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.3423 - mean_squared_error: 0.3423 - val_loss: 0.3618 - val_mean_squared_error: 0.3618
5160/1 [===================================================] - 0s 23us/sample - loss: 0.4712 - mean_squared_error: 0.3614
[0.36136142297308577, 0.3613615]

在这里，我们可以看到使用子类的方式来定义层次是比较繁琐和复杂的，如果我们只是想增加一个激活函数层，这样写就显得比较累赘了，所以我们可以使用lambda表达式来完成这个功能

import tensorflow as tf
from tensorflow.keras import layers, models
import numpy as np

if __name__ == "__main__":
    # 添加激活函数层sotgplus:log(1+e^x)
    customized_softplus = layers.Lambda(lambda x: tf.nn.softplus(x))
    print(customized_softplus([-10., -5., 0., 5., 10.]))
    model = models.Sequential([
        layers.Dense(30, activation='relu', input_shape=[None, 5]),
        layers.Dense(1),
        customized_softplus
        # 它等同于下面任意一行代码
        # layers.Dense(1, activation='softplus'),
        # layers.Dense(1), layers.Activation('softplus')
    ])
    print(model.summary())

运行结果

tf.Tensor([4.5417706e-05 6.7153489e-03 6.9314718e-01 5.0067153e+00 1.0000046e+01], shape=(5,), dtype=float32)
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, None, 30)          180       
_________________________________________________________________
dense_1 (Dense)              (None, None, 1)           31        
_________________________________________________________________
lambda (Lambda)              (None, None, 1)           0         
=================================================================
Total params: 211
Trainable params: 211
Non-trainable params: 0
_________________________________________________________________
None

tf.function函数转换图结构

这里我们有一个概念，那就是tensorflow的类型(常量，变量)以及它的函数都是被优化过，更加适合于使用GPU、TPU进行加速，那么对于普通的Python库函数是没有这个功能的。那么对于普通的Python库函数能否使用tensorflow的这种优化呢，答案是肯定的。

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import timeit
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    def scaled_elu(z, scale=1.0, alpha=1.0):
        # z >= 0 ? scale * z : scale * alpha * tf.nn.elu(z)
        is_positive = tf.greater_equal(z, 0.0)
        return scale * tf.where(is_positive, z, alpha * tf.nn.elu(z))

    t1 = tf.constant(-3.)
    t2 = tf.constant([-3., -2.5])
    print(scaled_elu(t1))
    print(scaled_elu(t2))
    scaled_elu_tf = tf.function(scaled_elu)
    print(scaled_elu_tf(t1))
    print(scaled_elu_tf(t2))
    print(scaled_elu_tf.python_function is scaled_elu)
    %timeit scaled_elu(tf.random.normal((1000, 1000)))
    %timeit scaled_elu_tf(tf.random.normal((1000, 1000)))

运行结果

tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
True
10 loops, best of 5: 25.8 ms per loop
10 loops, best of 5: 23.7 ms per loop

这里我们可以把scaled_elu看成是一个普通的Python函数，当我们对这个函数进行了图模型化scaled_elu_tf = tf.function(scaled_elu)后，当我们对其进行大型矩阵计算中，我们可以看到图模型化后的scaled_elu_tf比之前scaled_elu快了一点点。不过这里必须使用TPU环境才有效果，否则是比较慢的。除了这种直接转的方式，我们还可以通过打标签的方式

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import timeit
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    def scaled_elu(z, scale=1.0, alpha=1.0):
        # z >= 0 ? scale * z : scale * alpha * tf.nn.elu(z)
        is_positive = tf.greater_equal(z, 0.0)
        return scale * tf.where(is_positive, z, alpha * tf.nn.elu(z))

    t1 = tf.constant(-3.)
    t2 = tf.constant([-3., -2.5])
    print(scaled_elu(t1))
    print(scaled_elu(t2))
    scaled_elu_tf = tf.function(scaled_elu)
    print(scaled_elu_tf(t1))
    print(scaled_elu_tf(t2))
    print(scaled_elu_tf.python_function is scaled_elu)
    %timeit scaled_elu(tf.random.normal((1000, 1000)))
    %timeit scaled_elu_tf(tf.random.normal((1000, 1000)))

    @tf.function
    def converge_to_2(n_iters):
        # 1 + 1/2 + 1/2^2 + ... + 1/2^n
        total = tf.constant(0.)
        increment = tf.constant(1.)
        for _ in range(n_iters):
            total += increment
            increment /= 2.0
        return total

    print(converge_to_2(20))

运行结果

tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
True
10 loops, best of 5: 26 ms per loop
10 loops, best of 5: 24 ms per loop
tf.Tensor(1.9999981, shape=(), dtype=float32)

我们可以来看一下这两个函数转成图结构后的代码

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import timeit
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from IPython.display import display, Markdown

if __name__ == "__main__":

    def scaled_elu(z, scale=1.0, alpha=1.0):
        # z >= 0 ? scale * z : scale * alpha * tf.nn.elu(z)
        is_positive = tf.greater_equal(z, 0.0)
        return scale * tf.where(is_positive, z, alpha * tf.nn.elu(z))

    t1 = tf.constant(-3.)
    t2 = tf.constant([-3., -2.5])
    print(scaled_elu(t1))
    print(scaled_elu(t2))
    scaled_elu_tf = tf.function(scaled_elu)
    print(scaled_elu_tf(t1))
    print(scaled_elu_tf(t2))
    print(scaled_elu_tf.python_function is scaled_elu)
    %timeit scaled_elu(tf.random.normal((1000, 1000)))
    %timeit scaled_elu_tf(tf.random.normal((1000, 1000)))

    # @tf.function
    def converge_to_2(n_iters):
        # 1 + 1/2 + 1/2^2 + ... + 1/2^n
        total = tf.constant(0.)
        increment = tf.constant(1.)
        for _ in range(n_iters):
            total += increment
            increment /= 2.0
        return total

    print(converge_to_2(20))

    def display_tf_code(func):
        code = tf.autograph.to_code(func)
        display(Markdown('```python\n{}\n```'.format(code)))

    display_tf_code(scaled_elu)
    display_tf_code(converge_to_2)

运行结果

tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
tf.Tensor(-0.95021296, shape=(), dtype=float32)
tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)
True
10 loops, best of 5: 25.7 ms per loop
10 loops, best of 5: 23.9 ms per loop
tf.Tensor(1.9999981, shape=(), dtype=float32)
def tf__scaled_elu(z, scale=None, alpha=None):
    with ag__.FunctionScope('scaled_elu', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:
        do_return = False
        retval_ = ag__.UndefinedReturnValue()
        is_positive = ag__.converted_call(ag__.ld(tf).greater_equal, (ag__.ld(z), 0.0), None, fscope)
        try:
            do_return = True
            retval_ = (ag__.ld(scale) * ag__.converted_call(ag__.ld(tf).where, (ag__.ld(is_positive), ag__.ld(z), (ag__.ld(alpha) * ag__.converted_call(ag__.ld(tf).nn.elu, (ag__.ld(z),), None, fscope))), None, fscope))
        except:
            do_return = False
            raise
        return fscope.ret(retval_, do_return)
def tf__converge_to(n_iters):
    with ag__.FunctionScope('converge_to_2', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:
        do_return = False
        retval_ = ag__.UndefinedReturnValue()
        total = ag__.converted_call(ag__.ld(tf).constant, (0.0,), None, fscope)
        increment = ag__.converted_call(ag__.ld(tf).constant, (1.0,), None, fscope)

        def get_state():
            return (total, increment)

        def set_state(vars_):
            nonlocal increment, total
            (total, increment) = vars_

        def loop_body(itr):
            nonlocal increment, total
            _ = itr
            total = ag__.ld(total)
            total += increment
            increment = ag__.ld(increment)
            increment /= 2.0
        _ = ag__.Undefined('_')
        ag__.for_stmt(ag__.converted_call(ag__.ld(range), (ag__.ld(n_iters),), None, fscope), None, loop_body, get_state, set_state, ('total', 'increment'), {'iterate_names': '_'})
        try:
            do_return = True
            retval_ = ag__.ld(total)
        except:
            do_return = False
            raise
        return fscope.ret(retval_, do_return)

之前我们使用的都是常量，现在我们来看看对于变量来说，是什么情况

def add_21():
        var = tf.Variable(0.)
        return var.assign_add(21) # +=

print(add_21())

运行结果

<tf.Variable 'UnreadVariable' shape=() dtype=float32, numpy=21.0>

现在我们给这段代码加上标签

@tf.function
def add_21():
        var = tf.Variable(0.)
        return var.assign_add(21) # +=

print(add_21())

运行结果

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-8-806807c51c7b> in <module>()
      4         return var.assign_add(21) # +=
      5 
----> 6 print(add_21())

7 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
    992           except Exception as e:  # pylint:disable=broad-except
    993             if hasattr(e, "ag_error_metadata"):
--> 994               raise e.ag_error_metadata.to_exception(e)
    995             else:
    996               raise

ValueError: in user code:

    <ipython-input-8-806807c51c7b>:3 add_21  *
        var = tf.Variable(0.)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py:268 __call__  **
        return cls._variable_v2_call(*args, **kwargs)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py:262 _variable_v2_call
        shape=shape)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py:67 getter
        return captured_getter(captured_previous, **kwargs)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py:765 invalid_creator_scope
        "tf.function-decorated function tried to create "

    ValueError: tf.function-decorated function tried to create variables on non-first call.

它会直接报错，这个主要是对于tensorflow的图结构来说，它是不能直接在函数内部定义变量的，所以我们需要将变量定义到函数外部。

var = tf.Variable(0.)

@tf.function
def add_21():
        return var.assign_add(21) # +=

print(add_21())

运行结果

tf.Tensor(21.0, shape=(), dtype=float32)

由于Python是一个弱类型的语言，有的时候它可以输入多种不同的类型，现在我们要给图结构函数限制输入的类型

@tf.function(input_signature=[tf.TensorSpec([None], tf.int32, name='x')])
def cube(z):
    return tf.pow(z, 3)

print(cube(tf.constant([1., 2., 3.])))
print(cube(tf.constant([1, 2, 3])))

运行结果

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-10-53f91d549b69> in <module>()
      3     return tf.pow(z, 3)
      4 
----> 5 print(cube(tf.constant([1., 2., 3.])))
      6 print(cube(tf.constant([1, 2, 3])))

3 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py in _convert_inputs_to_signature(inputs, input_signature, flat_input_signature)
   2890       flatten_inputs)):
   2891     raise ValueError("Python inputs incompatible with input_signature:\n%s" %
-> 2892                      format_error_message(inputs, input_signature))
   2893 
   2894   if need_packing:

ValueError: Python inputs incompatible with input_signature:
  inputs: (
    tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32))
  input_signature: (
    TensorSpec(shape=(None,), dtype=tf.int32, name='x'))

这里我们给输入的参数限定为int32的整型，当我们输入整型的时候

@tf.function(input_signature=[tf.TensorSpec([None], tf.int32, name='x')])
def cube(z):
    return tf.pow(z, 3)

# print(cube(tf.constant([1., 2., 3.])))
print(cube(tf.constant([1, 2, 3])))

运行结果

tf.Tensor([ 1  8 27], shape=(3,), dtype=int32)

现在我们来看一下这个图的一些信息

# get_concrete_function -> 添加输入签名 ->保存模型
cube_func_int32 = cube.get_concrete_function(tf.TensorSpec([None], tf.int32))
print(cube_func_int32)
print(cube_func_int32 is cube.get_concrete_function(tf.TensorSpec([5], tf.int32)))
print(cube_func_int32 is cube.get_concrete_function(tf.constant([1, 2, 3])))

运行结果

ConcreteFunction cube(z)
  Args:
    z: int32 Tensor, shape=(None,)
  Returns:
    int32 Tensor, shape=(None,)
True
True

在cube_func_int32对象中就有图的信息

print(cube_func_int32.graph)
# 查看图中的操作
print(cube_func_int32.graph.get_operations())
# 查看立方操作
pow_op = cube_func_int32.graph.get_operations()[2]
print(pow_op)
# 查看该操作的输入和输出
print(list(pow_op.inputs))
print(list(pow_op.outputs))
# 通过操作名字获取操作
print(cube_func_int32.graph.get_operation_by_name("x"))
# 通过张量名字获取操作
print(cube_func_int32.graph.get_tensor_by_name("x:0"))
# 打印图定义
print(cube_func_int32.graph.as_graph_def())

运行结果

FuncGraph(name=cube, id=140354824183312)
[<tf.Operation 'x' type=Placeholder>, <tf.Operation 'Pow/y' type=Const>, <tf.Operation 'Pow' type=Pow>, <tf.Operation 'Identity' type=Identity>]
name: "Pow"
op: "Pow"
input: "x"
input: "Pow/y"
attr {
  key: "T"
  value {
    type: DT_INT32
  }
}

[<tf.Tensor 'x:0' shape=(None,) dtype=int32>, <tf.Tensor 'Pow/y:0' shape=() dtype=int32>]
[<tf.Tensor 'Pow:0' shape=(None,) dtype=int32>]
name: "x"
op: "Placeholder"
attr {
  key: "_user_specified_name"
  value {
    s: "x"
  }
}
attr {
  key: "dtype"
  value {
    type: DT_INT32
  }
}
attr {
  key: "shape"
  value {
    shape {
      dim {
        size: -1
      }
    }
  }
}

Tensor("x:0", shape=(None,), dtype=int32)
node {
  name: "x"
  op: "Placeholder"
  attr {
    key: "_user_specified_name"
    value {
      s: "x"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: -1
        }
      }
    }
  }
}
node {
  name: "Pow/y"
  op: "Const"
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 3
      }
    }
  }
}
node {
  name: "Pow"
  op: "Pow"
  input: "x"
  input: "Pow/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "Identity"
  op: "Identity"
  input: "Pow"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
}
versions {
  producer: 808
}

我们这里可以看到该图中有4个操作，这里有Placeholder的概念，Placeholder是tensorflow1中经常用到的图，可以显示定义，在2.0中就不需要了，它是用来放输入的，必须要放进去一个输入才能得到一个输出。那么图信息对我们有什么用呢？一般会用在两个地方，第一个地方就是如何去保存模型；第二个地方就是一旦保存了模型之后，如何把一个模型给载入进来，然后我们载入模型去做infos的时候会经常用到上面这些函数。