Merge pull request #18 from NREL/gb/dropout_bug

Gb/dropout bug

phygnn/phygnn.py

@@ -10,7 +10,7 @@
 import logging
 import tensorflow as tf
 from tensorflow.keras import optimizers, initializers
-from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.layers import BatchNormalization, Dropout
 
 from phygnn.utilities.loss_metrics import METRICS
 from phygnn.utilities.tf_layers import Layers
@@ -769,7 +769,7 @@ def predict(self, x, to_numpy=True, training=False):
             Flag to convert output from tensor to numpy array
         training : bool
             Flag for predict() used in the training routine. This is used
-            to freeze the BatchNormalization layers.
+            to freeze the BatchNormalization and Dropout layers.
 
         Returns
         -------
@@ -783,7 +783,7 @@ def predict(self, x, to_numpy=True, training=False):
         y = self.layers[0](x)
 
         for layer in self.layers[1:]:
-            if isinstance(layer, BatchNormalization):
+            if isinstance(layer, (BatchNormalization, Dropout)):
                 y = layer(y, training=training)
             else:
                 y = layer(y)

phygnn/version.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 """Physics Guided Neural Network version."""
 
-__version__ = '0.0.6'
+__version__ = '0.0.7'

tests/test_phygnn.py

@@ -282,31 +282,52 @@ def test_bad_pfun():
 
 def test_dropouts():
     """Test the dropout rate kwargs for adding dropout layers."""
-    HIDDEN_LAYERS = [
+    hidden_layers_1 = [
+        {'units': 64}, {'activation': 'relu'},
+        {'units': 64, 'activation': 'relu', 'name': 'relu2'}]
+    hidden_layers_2 = [
         {'units': 64}, {'activation': 'relu'}, {'dropout': 0.1},
         {'units': 64, 'activation': 'relu', 'name': 'relu2', 'dropout': 0.1}]
-    model = PhysicsGuidedNeuralNetwork(p_fun=p_fun_pythag,
-                                       hidden_layers=HIDDEN_LAYERS,
-                                       loss_weights=(0.0, 1.0),
-                                       n_features=2, n_labels=1)
+    PhysicsGuidedNeuralNetwork.seed()
+    model_1 = PhysicsGuidedNeuralNetwork(p_fun=p_fun_pythag,
+                                         hidden_layers=hidden_layers_1,
+                                         loss_weights=(0.0, 1.0),
+                                         n_features=2, n_labels=1)
+    PhysicsGuidedNeuralNetwork.seed()
+    model_2 = PhysicsGuidedNeuralNetwork(p_fun=p_fun_pythag,
+                                         hidden_layers=hidden_layers_2,
+                                         loss_weights=(0.0, 1.0),
+                                         n_features=2, n_labels=1)
 
-    assert len(model.layers) == 8, "dropout layers did not get added!"
-    assert isinstance(model.layers[0], InputLayer)
-    assert isinstance(model.layers[1], Dense)
-    assert isinstance(model.layers[2], Activation)
-    assert isinstance(model.layers[3], Dropout)
-    assert isinstance(model.layers[4], Dense)
-    assert isinstance(model.layers[5], Activation)
-    assert isinstance(model.layers[6], Dropout)
+    assert len(model_1.layers) == 6
+    assert len(model_2.layers) == 8, "dropout layers did not get added!"
+    assert isinstance(model_2.layers[0], InputLayer)
+    assert isinstance(model_2.layers[1], Dense)
+    assert isinstance(model_2.layers[2], Activation)
+    assert isinstance(model_2.layers[3], Dropout)
+    assert isinstance(model_2.layers[4], Dense)
+    assert isinstance(model_2.layers[5], Activation)
+    assert isinstance(model_2.layers[6], Dropout)
+
+    PhysicsGuidedNeuralNetwork.seed()
+    model_1.fit(X, Y_NOISE, P, n_batch=4, n_epoch=20)
 
-    model.fit(X, Y_NOISE, P, n_batch=4, n_epoch=20)
-    y_pred = model.predict(X)
+    PhysicsGuidedNeuralNetwork.seed()
+    model_2.fit(X, Y_NOISE, P, n_batch=4, n_epoch=20)
 
-    model.save(FPATH)
+    y_pred_1 = model_1.predict(X)
+    y_pred_2 = model_2.predict(X)
+
+    # make sure dropouts dont predict the same as non-dropout
+    diff = np.abs(y_pred_1 - y_pred_2)
+    assert not np.allclose(y_pred_1, y_pred_2)
+    assert np.max(diff) > 0.1
+
+    model_2.save(FPATH)
     loaded = PhysicsGuidedNeuralNetwork.load(FPATH)
     y_pred_loaded = loaded.predict(X)
-    assert np.allclose(y_pred, y_pred_loaded)
-    assert len(model.layers) == len(loaded.layers)
+    assert np.allclose(y_pred_2, y_pred_loaded)
+    assert len(model_2.layers) == len(loaded.layers)
     os.remove(FPATH)
 
 

tests/test_tf_model.py

@@ -100,6 +100,30 @@ def test_complex_nn():
     assert test_mae < loss
 
 
+def test_dropout():
+    """Test a model trained with dropout vs. no dropout and make sure the
+    predictions are different."""
+    hidden_layers_1 = [{'units': 64, 'activation': 'relu'},
+                       {'units': 64}, {'activation': 'relu'}]
+    hidden_layers_2 = [{'units': 64, 'activation': 'relu', 'dropout': 0.05},
+                       {'units': 64}, {'activation': 'relu'},
+                       {'dropout': 0.05}]
+    TfModel.seed()
+    model_1 = TfModel.build_trained(FEATURES, LABELS,
+                                    hidden_layers=hidden_layers_1,
+                                    epochs=10, fit_kwargs={"batch_size": 16},
+                                    early_stop=False)
+    TfModel.seed()
+    model_2 = TfModel.build_trained(FEATURES, LABELS,
+                                    hidden_layers=hidden_layers_2,
+                                    epochs=10, fit_kwargs={"batch_size": 16},
+                                    early_stop=False)
+
+    out1 = model_1.history['val_mae'].values[-5:]
+    out2 = model_2.history['val_mae'].values[-5:]
+    assert (out2 > out1).all()
+
+
 def test_save_load():
     """Test the save/load operations of TfModel"""
     hidden_layers = [{'units': 64, 'activation': 'relu', 'name': 'relu1'},