Skip to content
GitLab
Select Git revision
  • master default
  • enhancements
Find file BlameHistoryPermalink
Forked from Cresson Remi / SimpleExtractionTools
Source project has a limited visibility.
otbTensorflowModelTrain.cxx 24.32 KiB
1 
/*=========================================================================
2
3 
     Copyright (c) 2018-2019 IRSTEA
4 
     Copyright (c) 2020-2021 INRAE
5
6
7 
     This software is distributed WITHOUT ANY WARRANTY; without even
8 
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
9 
     PURPOSE.  See the above copyright notices for more information.
10
11 
=========================================================================*/
12 
#include "itkFixedArray.h"
13 
#include "itkObjectFactory.h"
14 
#include "otbWrapperApplicationFactory.h"
15
16 
// Application engine
17 
#include "otbStandardFilterWatcher.h"
18 
#include "itkFixedArray.h"
19
20 
// Tensorflow SavedModel
21 
#include "tensorflow/cc/saved_model/loader.h"
22
23 
// Tensorflow model train
24 
#include "otbTensorflowMultisourceModelTrain.h"
25 
#include "otbTensorflowMultisourceModelValidate.h"
26
27 
// Tensorflow graph load
28 
#include "otbTensorflowGraphOperations.h"
29
30 
// Layerstack
31 
#include "otbTensorflowSource.h"
32
33 
// Metrics
34 
#include "otbConfusionMatrixMeasurements.h"
35
36 
namespace otb
37 
{
38
39 
namespace Wrapper
40 
{
41
42 
class TensorflowModelTrain : public Application
43 
{
44 
public:
45
46 
  /** Standard class typedefs. */
47 
  typedef TensorflowModelTrain                       Self;
48 
  typedef Application                                Superclass;
49 
  typedef itk::SmartPointer<Self>                    Pointer;
50 
  typedef itk::SmartPointer<const Self>              ConstPointer;
51
52 
  /** Standard macro */
53 
  itkNewMacro(Self);
54 
  itkTypeMacro(TensorflowModelTrain, Application);
55
56 
  /** Typedefs for TensorFlow */
57 
  typedef otb::TensorflowMultisourceModelTrain<FloatVectorImageType>    TrainModelFilterType;
58 
  typedef otb::TensorflowMultisourceModelValidate<FloatVectorImageType> ValidateModelFilterType;
59 
  typedef otb::TensorflowSource<FloatVectorImageType>                   TFSource;
60
61 
  /* Typedefs for evaluation metrics */
62 
  typedef ValidateModelFilterType::ConfMatType                          ConfMatType;
63 
  typedef ValidateModelFilterType::MapOfClassesType                     MapOfClassesType;
64 
  typedef ValidateModelFilterType::LabelValueType                       LabelValueType;
65 
  typedef otb::ConfusionMatrixMeasurements<ConfMatType, LabelValueType> ConfusionMatrixCalculatorType;
66
67 
  //
68 
  // Store stuff related to one source
69 
  //
70 
  struct ProcessObjectsBundle
7172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
{ TFSource tfSource; TFSource tfSourceForValidation; // Parameters keys std::string m_KeyInForTrain; // Key of input image list (training) std::string m_KeyInForValid; // Key of input image list (validation) std::string m_KeyPHNameForTrain; // Key for placeholder name in the TensorFlow model (training) std::string m_KeyPHNameForValid; // Key for placeholder name in the TensorFlow model (validation) std::string m_KeyPszX; // Key for samples sizes X std::string m_KeyPszY; // Key for samples sizes Y }; /** Typedefs for the app */ typedef std::vector<ProcessObjectsBundle> BundleList; typedef std::vector<FloatVectorImageType::SizeType> SizeList; typedef std::vector<std::string> StringList; void DoUpdateParameters() { } // // Add an input source, which includes: // -an input image list (for training) // -an input image placeholder (for training) // -an input image list (for validation) // -an input image placeholder (for validation) // -an input patchsize, which is the dimensions of samples. Same for training and validation. // void AddAnInputImage() { // Number of source unsigned int inputNumber = m_Bundles.size() + 1; // Create keys and descriptions std::stringstream ss_key_tr_group, ss_desc_tr_group, ss_key_val_group, ss_desc_val_group, ss_key_tr_in, ss_desc_tr_in, ss_key_val_in, ss_desc_val_in, ss_key_dims_x, ss_desc_dims_x, ss_key_dims_y, ss_desc_dims_y, ss_key_tr_ph, ss_desc_tr_ph, ss_key_val_ph, ss_desc_val_ph; // Parameter group key/description ss_key_tr_group << "training.source" << inputNumber; ss_key_val_group << "validation.source" << inputNumber; ss_desc_tr_group << "Parameters for source #" << inputNumber << " (training)"; ss_desc_val_group << "Parameters for source #" << inputNumber << " (validation)"; // Parameter group keys ss_key_tr_in << ss_key_tr_group.str() << ".il"; ss_key_val_in << ss_key_val_group.str() << ".il"; ss_key_dims_x << ss_key_tr_group.str() << ".patchsizex"; ss_key_dims_y << ss_key_tr_group.str() << ".patchsizey"; ss_key_tr_ph << ss_key_tr_group.str() << ".placeholder"; ss_key_val_ph << ss_key_val_group.str() << ".name"; // Parameter group descriptions ss_desc_tr_in << "Input image (or list to stack) for source #" << inputNumber << " (training)"; ss_desc_val_in << "Input image (or list to stack) for source #" << inputNumber << " (validation)"; ss_desc_dims_x << "Patch size (x) for source #" << inputNumber; ss_desc_dims_y << "Patch size (y) for source #" << inputNumber; ss_desc_tr_ph << "Name of the input placeholder for source #" << inputNumber << " (training)"; ss_desc_val_ph << "Name of the input placeholder " "or output tensor for source #" << inputNumber << " (validation)"; // Populate group AddParameter(ParameterType_Group, ss_key_tr_group.str(), ss_desc_tr_group.str());
141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
AddParameter(ParameterType_InputImageList, ss_key_tr_in.str(), ss_desc_tr_in.str() ); AddParameter(ParameterType_Int, ss_key_dims_x.str(), ss_desc_dims_x.str()); SetMinimumParameterIntValue (ss_key_dims_x.str(), 1); AddParameter(ParameterType_Int, ss_key_dims_y.str(), ss_desc_dims_y.str()); SetMinimumParameterIntValue (ss_key_dims_y.str(), 1); AddParameter(ParameterType_String, ss_key_tr_ph.str(), ss_desc_tr_ph.str()); AddParameter(ParameterType_Group, ss_key_val_group.str(), ss_desc_val_group.str()); AddParameter(ParameterType_InputImageList, ss_key_val_in.str(), ss_desc_val_in.str() ); AddParameter(ParameterType_String, ss_key_val_ph.str(), ss_desc_val_ph.str()); // Add a new bundle ProcessObjectsBundle bundle; bundle.m_KeyInForTrain = ss_key_tr_in.str(); bundle.m_KeyInForValid = ss_key_val_in.str(); bundle.m_KeyPHNameForTrain = ss_key_tr_ph.str(); bundle.m_KeyPHNameForValid = ss_key_val_ph.str(); bundle.m_KeyPszX = ss_key_dims_x.str(); bundle.m_KeyPszY = ss_key_dims_y.str(); m_Bundles.push_back(bundle); } void DoInit() { // Documentation SetName("TensorflowModelTrain"); SetDescription("Train a multisource deep learning net using Tensorflow. Change " "the " + tf::ENV_VAR_NAME_NSOURCES + " environment variable to set the number of " "sources."); SetDocLongDescription("The application trains a Tensorflow model over multiple data sources. " "The number of input sources can be changed at runtime by setting the " "system environment variable " + tf::ENV_VAR_NAME_NSOURCES + ". " "For each source, you have to set (1) the tensor placeholder name, as named in " "the tensorflow model, (2) the patch size and (3) the image(s) source. "); SetDocAuthors("Remi Cresson"); AddDocTag(Tags::Learning); // Input model AddParameter(ParameterType_Group, "model", "Model parameters"); AddParameter(ParameterType_Directory, "model.dir", "Tensorflow model_save directory"); MandatoryOn ("model.dir"); AddParameter(ParameterType_String, "model.restorefrom", "Restore model from path"); MandatoryOff ("model.restorefrom"); AddParameter(ParameterType_String, "model.saveto", "Save model to path"); MandatoryOff ("model.saveto"); AddParameter(ParameterType_StringList, "model.tagsets", "Which tags (i.e. v1.MetaGraphDefs) to load from the saved model. Currently, only one tag is supported. Can be retrieved by running `saved_model_cli show --dir your_model_dir --all`"); MandatoryOff ("model.tagsets"); // Training parameters group AddParameter(ParameterType_Group, "training", "Training parameters"); AddParameter(ParameterType_Int, "training.batchsize", "Batch size"); SetMinimumParameterIntValue ("training.batchsize", 1); SetDefaultParameterInt ("training.batchsize", 100); AddParameter(ParameterType_Int, "training.epochs", "Number of epochs"); SetMinimumParameterIntValue ("training.epochs", 1); SetDefaultParameterInt ("training.epochs", 100); AddParameter(ParameterType_StringList, "training.userplaceholders", "Additional single-valued placeholders for training. Supported types: int, float, bool."); MandatoryOff ("training.userplaceholders"); AddParameter(ParameterType_StringList, "training.targetnodes", "Names of the target nodes"); MandatoryOn ("training.targetnodes"); AddParameter(ParameterType_StringList, "training.outputtensors", "Names of the output tensors to display"); MandatoryOff ("training.outputtensors"); AddParameter(ParameterType_Bool, "training.usestreaming", "Use the streaming through patches (slower but can process big dataset)"); MandatoryOff ("training.usestreaming"); // Metrics AddParameter(ParameterType_Group, "validation", "Validation parameters");
211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
MandatoryOff ("validation"); AddParameter(ParameterType_Int, "validation.step", "Perform the validation every Nth epochs"); SetMinimumParameterIntValue ("validation.step", 1); SetDefaultParameterInt ("validation.step", 10); AddParameter(ParameterType_Choice, "validation.mode", "Metrics to compute"); AddChoice ("validation.mode.none", "No validation step"); AddChoice ("validation.mode.class", "Classification metrics"); AddChoice ("validation.mode.rmse", "Root mean square error"); AddParameter(ParameterType_StringList, "validation.userplaceholders", "Additional single-valued placeholders for validation. Supported types: int, float, bool."); MandatoryOff ("validation.userplaceholders"); AddParameter(ParameterType_Bool, "validation.usestreaming", "Use the streaming through patches (slower but can process big dataset)"); MandatoryOff ("validation.usestreaming"); // Input/output images AddAnInputImage(); for (int i = 1; i < tf::GetNumberOfSources() + 1 ; i++) // +1 because we have at least 1 source more for training { AddAnInputImage(); } // Example SetDocExampleParameterValue("source1.il", "spot6pms.tif"); SetDocExampleParameterValue("source1.placeholder", "x1"); SetDocExampleParameterValue("source1.patchsizex", "16"); SetDocExampleParameterValue("source1.patchsizey", "16"); SetDocExampleParameterValue("source2.il", "labels.tif"); SetDocExampleParameterValue("source2.placeholder", "y1"); SetDocExampleParameterValue("source2.patchsizex", "1"); SetDocExampleParameterValue("source2.patchsizex", "1"); SetDocExampleParameterValue("model.dir", "/tmp/my_saved_model/"); SetDocExampleParameterValue("training.userplaceholders", "is_training=true dropout=0.2"); SetDocExampleParameterValue("training.targetnodes", "optimizer"); SetDocExampleParameterValue("model.saveto", "/tmp/my_saved_model/variables/variables"); } // // Prepare bundles // Here, we populate the two following groups: // 1.Training : // -Placeholders // -PatchSize // -ImageSource // 2.Learning/Validation // -Placeholders (if input) or Tensor name (if target) // -PatchSize (which is the same as for training) // -ImageSource (depending if it's for learning or validation) // // TODO: a bit of refactoring. We could simply rely on m_Bundles // if we can keep trace of indices of sources for // training / test / validation // void PrepareInputs() { // Clear placeholder names m_InputPlaceholdersForTraining.clear(); m_InputPlaceholdersForValidation.clear(); // Clear patches sizes m_InputPatchesSizeForTraining.clear(); m_InputPatchesSizeForValidation.clear(); m_TargetPatchesSize.clear(); // Clear bundles m_InputSourcesForTraining.clear(); m_InputSourcesForEvaluationAgainstLearningData.clear(); m_InputSourcesForEvaluationAgainstValidationData.clear(); m_TargetTensorsNames.clear();
281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350
m_InputTargetsForEvaluationAgainstValidationData.clear(); m_InputTargetsForEvaluationAgainstLearningData.clear(); // Prepare the bundles for (auto& bundle: m_Bundles) { // Source FloatVectorImageListType::Pointer trainStack = GetParameterImageList(bundle.m_KeyInForTrain); bundle.tfSource.Set(trainStack); m_InputSourcesForTraining.push_back(bundle.tfSource.Get()); // Placeholder std::string placeholderForTraining = GetParameterAsString(bundle.m_KeyPHNameForTrain); m_InputPlaceholdersForTraining.push_back(placeholderForTraining); // Patch size FloatVectorImageType::SizeType patchSize; patchSize[0] = GetParameterInt(bundle.m_KeyPszX); patchSize[1] = GetParameterInt(bundle.m_KeyPszY); m_InputPatchesSizeForTraining.push_back(patchSize); otbAppLogINFO("New source:"); otbAppLogINFO("Patch size : "<< patchSize); otbAppLogINFO("Placeholder (training) : "<< placeholderForTraining); // Prepare validation sources if (GetParameterInt("validation.mode") != 0) { // Get the stack if (!HasValue(bundle.m_KeyInForValid)) { otbAppLogFATAL("No validation input is set for this source"); } FloatVectorImageListType::Pointer validStack = GetParameterImageList(bundle.m_KeyInForValid); bundle.tfSourceForValidation.Set(validStack); // We check if the placeholder is the same for training and for validation // If yes, it means that its not an output tensor on which perform the validation std::string placeholderForValidation = GetParameterAsString(bundle.m_KeyPHNameForValid); if (placeholderForValidation.empty()) { placeholderForValidation = placeholderForTraining; } // Same placeholder name ==> is a source for validation if (placeholderForValidation.compare(placeholderForTraining) == 0) { // Source m_InputSourcesForEvaluationAgainstValidationData.push_back(bundle.tfSourceForValidation.Get()); m_InputSourcesForEvaluationAgainstLearningData.push_back(bundle.tfSource.Get()); // Placeholder m_InputPlaceholdersForValidation.push_back(placeholderForValidation); // Patch size m_InputPatchesSizeForValidation.push_back(patchSize); otbAppLogINFO("Placeholder (validation) : "<< placeholderForValidation); } // Different placeholder ==> is a target to validate else { // Source m_InputTargetsForEvaluationAgainstValidationData.push_back(bundle.tfSourceForValidation.Get()); m_InputTargetsForEvaluationAgainstLearningData.push_back(bundle.tfSource.Get()); // Placeholder m_TargetTensorsNames.push_back(placeholderForValidation);
351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420
// Patch size m_TargetPatchesSize.push_back(patchSize); otbAppLogINFO("Tensor name (validation) : "<< placeholderForValidation); } } } } // // Get user placeholders // TrainModelFilterType::DictType GetUserPlaceholders(const std::string & key) { TrainModelFilterType::DictType dict; TrainModelFilterType::StringList expressions = GetParameterStringList(key); for (auto& exp: expressions) { TrainModelFilterType::DictElementType entry = tf::ExpressionToTensor(exp); dict.push_back(entry); otbAppLogINFO("Using placeholder " << entry.first << " with " << tf::PrintTensorInfos(entry.second)); } return dict; } // // Print some classification metrics // void PrintClassificationMetrics(const ConfMatType & confMat, const MapOfClassesType & mapOfClassesRef) { ConfusionMatrixCalculatorType::Pointer confMatMeasurements = ConfusionMatrixCalculatorType::New(); confMatMeasurements->SetConfusionMatrix(confMat); confMatMeasurements->SetMapOfClasses(mapOfClassesRef); confMatMeasurements->Compute(); for (auto const& itMapOfClassesRef : mapOfClassesRef) { LabelValueType labelRef = itMapOfClassesRef.first; LabelValueType indexLabelRef = itMapOfClassesRef.second; otbAppLogINFO("Precision of class [" << labelRef << "] vs all: " << confMatMeasurements->GetPrecisions()[indexLabelRef]); otbAppLogINFO("Recall of class [" << labelRef << "] vs all: " << confMatMeasurements->GetRecalls()[indexLabelRef]); otbAppLogINFO("F-score of class [" << labelRef << "] vs all: " << confMatMeasurements->GetFScores()[indexLabelRef]); otbAppLogINFO("\t"); } otbAppLogINFO("Precision of the different classes: " << confMatMeasurements->GetPrecisions()); otbAppLogINFO("Recall of the different classes: " << confMatMeasurements->GetRecalls()); otbAppLogINFO("F-score of the different classes: " << confMatMeasurements->GetFScores()); otbAppLogINFO("\t"); otbAppLogINFO("Kappa index: " << confMatMeasurements->GetKappaIndex()); otbAppLogINFO("Overall accuracy index: " << confMatMeasurements->GetOverallAccuracy()); otbAppLogINFO("Confusion matrix:\n" << confMat); } void DoExecute() { // Load the Tensorflow bundle tf::LoadModel(GetParameterAsString("model.dir"), m_SavedModel, GetParameterStringList("model.tagsets")); // Check if we have to restore variables from somewhere else if (HasValue("model.restorefrom")) { const std::string path = GetParameterAsString("model.restorefrom"); otbAppLogINFO("Restoring model from " + path); // Load SavedModel variables
421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490
tf::RestoreModel(path, m_SavedModel); } // Prepare inputs PrepareInputs(); // Setup training filter m_TrainModelFilter = TrainModelFilterType::New(); m_TrainModelFilter->SetSavedModel(&m_SavedModel); m_TrainModelFilter->SetOutputTensors(GetParameterStringList("training.outputtensors")); m_TrainModelFilter->SetTargetNodesNames(GetParameterStringList("training.targetnodes")); m_TrainModelFilter->SetBatchSize(GetParameterInt("training.batchsize")); m_TrainModelFilter->SetUserPlaceholders(GetUserPlaceholders("training.userplaceholders")); m_TrainModelFilter->SetUseStreaming(GetParameterInt("training.usestreaming")); // Set inputs for (unsigned int i = 0 ; i < m_InputSourcesForTraining.size() ; i++) { m_TrainModelFilter->PushBackInputTensorBundle( m_InputPlaceholdersForTraining[i], m_InputPatchesSizeForTraining[i], m_InputSourcesForTraining[i]); } // Setup the validation filter const bool do_validation = HasUserValue("validation.mode"); if (GetParameterInt("validation.mode")==1) // class { otbAppLogINFO("Set validation mode to classification validation"); m_ValidateModelFilter = ValidateModelFilterType::New(); m_ValidateModelFilter->SetSavedModel(&m_SavedModel); m_ValidateModelFilter->SetBatchSize(GetParameterInt("training.batchsize")); m_ValidateModelFilter->SetUserPlaceholders(GetUserPlaceholders("validation.userplaceholders")); m_ValidateModelFilter->SetInputPlaceholders(m_InputPlaceholdersForValidation); m_ValidateModelFilter->SetInputReceptiveFields(m_InputPatchesSizeForValidation); m_ValidateModelFilter->SetOutputTensors(m_TargetTensorsNames); m_ValidateModelFilter->SetOutputExpressionFields(m_TargetPatchesSize); } else if (GetParameterInt("validation.mode")==2) // rmse) { otbAppLogINFO("Set validation mode to classification RMSE evaluation"); otbAppLogFATAL("Not implemented yet !"); // XD // TODO } // Epoch for (int epoch = 1 ; epoch <= GetParameterInt("training.epochs") ; epoch++) { // Train the model AddProcess(m_TrainModelFilter, "Training epoch #" + std::to_string(epoch)); m_TrainModelFilter->Update(); if (do_validation) { // Validate the model if (epoch % GetParameterInt("validation.step") == 0) { // 1. Evaluate the metrics against the learning data for (unsigned int i = 0 ; i < m_InputSourcesForEvaluationAgainstLearningData.size() ; i++) { m_ValidateModelFilter->SetInput(i, m_InputSourcesForEvaluationAgainstLearningData[i]); } m_ValidateModelFilter->SetInputReferences(m_InputTargetsForEvaluationAgainstLearningData); // As we use the learning data here, it's rational to use the same option as streaming during training m_ValidateModelFilter->SetUseStreaming(GetParameterInt("training.usestreaming"));
491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560
// Update AddProcess(m_ValidateModelFilter, "Evaluate model (Learning data)"); m_ValidateModelFilter->Update(); for (unsigned int i = 0 ; i < m_TargetTensorsNames.size() ; i++) { otbAppLogINFO("Metrics for target \"" << m_TargetTensorsNames[i] << "\":"); PrintClassificationMetrics(m_ValidateModelFilter->GetConfusionMatrix(i), m_ValidateModelFilter->GetMapOfClasses(i)); } // 2. Evaluate the metrics against the validation data // Here we just change the input sources and references for (unsigned int i = 0 ; i < m_InputSourcesForEvaluationAgainstValidationData.size() ; i++) { m_ValidateModelFilter->SetInput(i, m_InputSourcesForEvaluationAgainstValidationData[i]); } m_ValidateModelFilter->SetInputReferences(m_InputTargetsForEvaluationAgainstValidationData); m_ValidateModelFilter->SetUseStreaming(GetParameterInt("validation.usestreaming")); // Update AddProcess(m_ValidateModelFilter, "Evaluate model (Validation data)"); m_ValidateModelFilter->Update(); for (unsigned int i = 0 ; i < m_TargetTensorsNames.size() ; i++) { otbAppLogINFO("Metrics for target \"" << m_TargetTensorsNames[i] << "\":"); PrintClassificationMetrics(m_ValidateModelFilter->GetConfusionMatrix(i), m_ValidateModelFilter->GetMapOfClasses(i)); } } // Step is OK to perform validation } // Do the validation against the validation data } // Next epoch // Check if we have to save variables to somewhere if (HasValue("model.saveto")) { const std::string path = GetParameterAsString("model.saveto"); otbAppLogINFO("Saving model to " + path); tf::SaveModel(path, m_SavedModel); } } private: tensorflow::SavedModelBundle m_SavedModel; // must be alive during all the execution of the application ! // Filters TrainModelFilterType::Pointer m_TrainModelFilter; ValidateModelFilterType::Pointer m_ValidateModelFilter; // Inputs BundleList m_Bundles; // Patches size SizeList m_InputPatchesSizeForTraining; SizeList m_InputPatchesSizeForValidation; SizeList m_TargetPatchesSize; // Placeholders and Tensors names StringList m_InputPlaceholdersForTraining; StringList m_InputPlaceholdersForValidation; StringList m_TargetTensorsNames; // Image sources std::vector<FloatVectorImageType::Pointer> m_InputSourcesForTraining; std::vector<FloatVectorImageType::Pointer> m_InputSourcesForEvaluationAgainstLearningData; std::vector<FloatVectorImageType::Pointer> m_InputSourcesForEvaluationAgainstValidationData; std::vector<FloatVectorImageType::Pointer> m_InputTargetsForEvaluationAgainstLearningData;
561562563564565566567568569
std::vector<FloatVectorImageType::Pointer> m_InputTargetsForEvaluationAgainstValidationData; }; // end of class } // namespace wrapper } // namespace otb OTB_APPLICATION_EXPORT( otb::Wrapper::TensorflowModelTrain )