From c22b8e04d93fb6df0b0ea4771b71b6d2a24f09e2 Mon Sep 17 00:00:00 2001 From: "raffaele.gaetano" <raffaele.gaetano@cirad.fr> Date: Wed, 20 Jul 2022 15:30:43 +0200 Subject: [PATCH] Two separate apps for tile segmentation and graph aggregation. --- app/CMakeLists.txt | 8 ++ app/otbAssembleGRMGraphs.cxx | 227 +++++++++++++++++++++++++++++++++ app/otbSingleTileGRMGraph.cxx | 230 ++++++++++++++++++++++++++++++++++ include/lsgrmController.h | 4 + include/lsgrmController.txx | 111 ++++++++++++++++ 5 files changed, 580 insertions(+) create mode 100644 app/otbAssembleGRMGraphs.cxx create mode 100644 app/otbSingleTileGRMGraph.cxx diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 1c87c95..4257143 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -7,3 +7,11 @@ OTB_CREATE_APPLICATION(NAME LSGRM OTB_CREATE_APPLICATION(NAME SimpleVectorization SOURCES otbSimpleVectorization.cxx LINK_LIBRARIES ${${otb-module}_LIBRARIES} OTBGRM) + +OTB_CREATE_APPLICATION(NAME SingleTileGRMGraph + SOURCES otbSingleTileGRMGraph.cxx + LINK_LIBRARIES ${${otb-module}_LIBRARIES} OTBGRM) + +OTB_CREATE_APPLICATION(NAME AssembleGRMGraphs + SOURCES otbAssembleGRMGraphs.cxx + LINK_LIBRARIES ${${otb-module}_LIBRARIES} OTBGRM) diff --git a/app/otbAssembleGRMGraphs.cxx b/app/otbAssembleGRMGraphs.cxx new file mode 100644 index 0000000..ec63f06 --- /dev/null +++ b/app/otbAssembleGRMGraphs.cxx @@ -0,0 +1,227 @@ +#include "itkFixedArray.h" +#include "itkObjectFactory.h" + +// Elevation handler +#include "otbWrapperElevationParametersHandler.h" +#include "otbWrapperApplicationFactory.h" + +// Application engine +#include "otbStandardFilterWatcher.h" +#include "itkFixedArray.h" +#include "itkImageSource.h" + +// LSGRM +#include <iostream> +#include "lsgrmBaatzSegmenter.h" +#include "lsgrmSpringSegmenter.h" +#include "lsgrmFullLambdaScheduleSegmenter.h" +#include "lsgrmController.h" +#include "lsgrmGraphOperations.h" + +// Graph to label image (streaming version) +#include "otbStreamingGraphToImageFilter.h" +#include "otbStreamingImageVirtualWriter.h" + +// system tools +#include <itksys/SystemTools.hxx> + +namespace otb +{ + +namespace Wrapper +{ + +class AssembleGRMGraphs : public Application +{ +public: + /** Standard class typedefs. */ + typedef AssembleGRMGraphs Self; + typedef Application Superclass; + typedef itk::SmartPointer<Self> Pointer; + typedef itk::SmartPointer<const Self> ConstPointer; + + /** Standard macro */ + itkNewMacro(Self); + itkTypeMacro(AssembleGRMGraphs, Application); + + /** Useful typedefs */ + typedef otb::VectorImage<float, 2> ImageType; + typedef lsgrm::BaatzSegmenter<ImageType> BaatzSegmenterType; + typedef lsgrm::SpringSegmenter<ImageType> SpringSegmenterType; + typedef lsgrm::FullLambdaScheduleSegmenter<ImageType> FLSSegmenterType; + +private: + + /* Tiling mode choice */ + enum TilingMode + { + TILING_AUTO, + TILING_USER, + TILING_NONE + }; + + /* Criterion choice */ + enum Criterion + { + CRITERION_BAATZ, + CRITERION_SPRING, + CRITERION_FLS + }; + + void DoInit() + { + + SetName("AssembleGRMGraphs"); + SetDescription("This application assembles a full set of graphs from a LSGRM layout" + "and provides the final segmentation image. The full-scale graph must fit in memory."); + + // Input and Output images + AddParameter(ParameterType_InputImage, "in", "Original input image to segment (for layout computation)"); + AddParameter(ParameterType_InputFilename, "graph", "Path and basename for the input set of graphs."); + AddParameter(ParameterType_OutputImage, "out", "Final segmentation image."); + + // Criterion choice + AddParameter(ParameterType_Choice, "criterion", "Homogeneity criterion to use"); + AddChoice("criterion.bs", "Baatz & Schape"); + AddChoice("criterion.ed", "Euclidean Distance"); + AddChoice("criterion.fls", "Full Lambda Schedule"); + + // Generic parameters + AddParameter(ParameterType_Float, "threshold", "Threshold for the criterion"); + SetDefaultParameterFloat("threshold", 100.0); + MandatoryOff("threshold"); + + // Specific parameters for Baatz & Schape + AddParameter(ParameterType_Float, "criterion.bs.cw", "Weight for the spectral homogeneity"); + SetDefaultParameterFloat("criterion.bs.cw", 0.5); + MandatoryOff("criterion.bs.cw"); + AddParameter(ParameterType_Float, "criterion.bs.sw", "Weight for the spatial homogeneity"); + SetDefaultParameterFloat("criterion.bs.sw", 0.5); + MandatoryOff("criterion.bs.sw"); + + // For large scale + AddParameter(ParameterType_Choice, "tiling", "Tiling layout used to compute the available set of graphs."); + AddChoice("tiling.auto", "Automatic tiling layout"); + AddChoice("tiling.user", "User tiling layout"); + AddParameter(ParameterType_Int, "tiling.user.sizex", "Tiles width"); + AddParameter(ParameterType_Int, "tiling.user.sizey", "Tiles height"); + + AddParameter(ParameterType_Int, "maxiter", "Maximum number of final iterations."); + SetDefaultParameterInt("maxiter", 10000); + MandatoryOff("maxiter"); + + } + + void DoUpdateParameters() + { + } + + /* + * This function sets the generic parameters of a controller and runs the segmentation + */ + template<class TSegmenter> + void + SetGenericParametersAndRunSegmentation(const typename TSegmenter::ParamType params){ + + // Instantiate the controller + typedef typename lsgrm::Controller<TSegmenter> ControlerType; + typename ControlerType::Pointer controller = ControlerType::New(); + using GraphType = typename ControlerType::GraphType; + + // Set specific parameters + controller->SetSpecificParameters(params); + + // Set input image + ImageType::Pointer inputImage = GetParameterFloatVectorImage("in"); + controller->SetInputImage(inputImage); + controller->SetTemporaryFilesPrefix(GetParameterString("graph")); + + // Set threshold + float thres = GetParameterFloat("threshold"); + controller->SetThreshold(thres*thres); + + // Switch tiling mode + int inputTilingMode = GetParameterInt("tiling"); + if (inputTilingMode == TILING_AUTO) + { + // Automatic mode + controller->SetTilingModeAuto(); + } + else if (inputTilingMode == TILING_USER) + { + // User mode + controller->SetTilingModeUser(); + controller->SetTileWidth(GetParameterInt("tiling.user.sizex")); + controller->SetTileHeight(GetParameterInt("tiling.user.sizey")); + } + else if (inputTilingMode == TILING_NONE) + { + // None mode + controller->SetTilingModeNone(); + } + else + { + otbAppLogFATAL("Unknown tiling mode!"); + } + + // Run the segmentation + controller->GetTilingLayout(); + controller->JustMergeAndAchieveSegmentation(GetParameterInt("maxiter")); + + UInt32ImageType::Pointer m_labelImage; + + // Prepare the label image source + typedef lsgrm::StreamingGraphToImageFilter<GraphType, UInt32ImageType> LabelImageSourceType; + typename LabelImageSourceType::Pointer labelImageSource = LabelImageSourceType::New(); + labelImageSource->SetGraph(controller->GetOutputGraph()); + labelImageSource->SetOutputSize(inputImage->GetLargestPossibleRegion().GetSize()); + labelImageSource->SetOutputOrigin(inputImage->GetOrigin()); + labelImageSource->SetOutputSpacing(inputImage->GetSignedSpacing()); + labelImageSource->SetOutputProjectionRef(inputImage->GetProjectionRef()); + labelImageSource->GenerateOutputInformation(); + + m_LabelImageSource = static_cast<itk::ImageSource<UInt32ImageType>*>(labelImageSource); + m_labelImage = m_LabelImageSource->GetOutput(); + SetParameterOutputImage<UInt32ImageType>("out", m_labelImage); + + } + + void DoExecute() + { + + ImageType::Pointer inputImage = GetParameterFloatVectorImage("in"); + + // Switch criterion + int inputCriterion = GetParameterInt("criterion"); + if (inputCriterion == CRITERION_BAATZ) + { + grm::BaatzParam params; + params.m_SpectralWeight = GetParameterFloat("criterion.bs.cw"); + params.m_ShapeWeight = GetParameterFloat("criterion.bs.sw"); + SetGenericParametersAndRunSegmentation<BaatzSegmenterType>(params); + } + else if (inputCriterion == CRITERION_SPRING) + { + grm::SpringParam params; + SetGenericParametersAndRunSegmentation<SpringSegmenterType>(params); + } + else if (inputCriterion == CRITERION_FLS) + { + grm::FLSParam params; + SetGenericParametersAndRunSegmentation<FLSSegmenterType>(params); + } + else + { + otbAppLogFATAL("Unknow criterion!") + } + + } + +private: + itk::ImageSource<UInt32ImageType>::Pointer m_LabelImageSource; + +}; // app class +} // end of namespace wrapper +} // end of namespace otb + +OTB_APPLICATION_EXPORT(otb::Wrapper::AssembleGRMGraphs) diff --git a/app/otbSingleTileGRMGraph.cxx b/app/otbSingleTileGRMGraph.cxx new file mode 100644 index 0000000..50b5633 --- /dev/null +++ b/app/otbSingleTileGRMGraph.cxx @@ -0,0 +1,230 @@ +#include "itkFixedArray.h" +#include "itkObjectFactory.h" + +// Elevation handler +#include "otbWrapperElevationParametersHandler.h" +#include "otbWrapperApplicationFactory.h" + +// Application engine +#include "otbStandardFilterWatcher.h" +#include "itkFixedArray.h" +#include "itkImageSource.h" + +// LSGRM +#include <iostream> +#include "lsgrmBaatzSegmenter.h" +#include "lsgrmSpringSegmenter.h" +#include "lsgrmFullLambdaScheduleSegmenter.h" +#include "lsgrmController.h" +#include "lsgrmGraphOperations.h" + +// Graph to label image (streaming version) +#include "otbStreamingGraphToImageFilter.h" +#include "otbStreamingImageVirtualWriter.h" + +// system tools +#include <itksys/SystemTools.hxx> + +namespace otb +{ + +namespace Wrapper +{ + +class SingleTileGRMGraph : public Application +{ +public: + /** Standard class typedefs. */ + typedef SingleTileGRMGraph Self; + typedef Application Superclass; + typedef itk::SmartPointer<Self> Pointer; + typedef itk::SmartPointer<const Self> ConstPointer; + + /** Standard macro */ + itkNewMacro(Self); + itkTypeMacro(SingleTileGRMGraph, Application); + + /** Useful typedefs */ + typedef otb::VectorImage<float, 2> ImageType; + typedef lsgrm::BaatzSegmenter<ImageType> BaatzSegmenterType; + typedef lsgrm::SpringSegmenter<ImageType> SpringSegmenterType; + typedef lsgrm::FullLambdaScheduleSegmenter<ImageType> FLSSegmenterType; + +private: + + /* Tiling mode choice */ + enum TilingMode + { + TILING_AUTO, + TILING_USER, + TILING_NONE + }; + + /* Criterion choice */ + enum Criterion + { + CRITERION_BAATZ, + CRITERION_SPRING, + CRITERION_FLS + }; + + void DoInit() + { + + SetName("SingleTileGRMGraph"); + SetDescription("This application provides the individual tile graphs from a LSGRM layout. " + "As LSGRM it provides currently 3 homogeneity criteria: Euclidean Distance, " + "Full Lambda Schedule and Baatz & Schape criterion."); + + // Input and Output images + AddParameter(ParameterType_InputImage, "in", "Input Image"); + AddParameter(ParameterType_Int, "xtileidx", "X index of tile to process"); + AddParameter(ParameterType_Int, "ytileidx", "Y index of tile to process"); + MandatoryOff("xtileidx"); + MandatoryOff("ytileidx"); + AddParameter(ParameterType_OutputFilename, "out", "Path and basename for the output graph."); + + // Criterion choice + AddParameter(ParameterType_Choice, "criterion", "Homogeneity criterion to use"); + AddChoice("criterion.bs", "Baatz & Schape"); + AddChoice("criterion.ed", "Euclidean Distance"); + AddChoice("criterion.fls", "Full Lambda Schedule"); + + // Generic parameters + AddParameter(ParameterType_Float, "threshold", "Threshold for the criterion"); + SetDefaultParameterFloat("threshold", 100.0); + MandatoryOff("threshold"); + + // Specific parameters for Baatz & Schape + AddParameter(ParameterType_Float, "criterion.bs.cw", "Weight for the spectral homogeneity"); + SetDefaultParameterFloat("criterion.bs.cw", 0.5); + MandatoryOff("criterion.bs.cw"); + AddParameter(ParameterType_Float, "criterion.bs.sw", "Weight for the spatial homogeneity"); + SetDefaultParameterFloat("criterion.bs.sw", 0.5); + MandatoryOff("criterion.bs.sw"); + + // For large scale + AddParameter(ParameterType_Choice, "tiling", "Tiling layout for the large scale segmentation"); + AddChoice("tiling.auto", "Automatic tiling layout"); + AddChoice("tiling.user", "User tiling layout"); + AddParameter(ParameterType_Int, "tiling.user.sizex", "Tiles width"); + AddParameter(ParameterType_Int, "tiling.user.sizey", "Tiles height"); + AddParameter(ParameterType_Int, "tiling.user.nfirstiter", "Number of first iterations"); + AddChoice("tiling.none", "No tiling layout"); + } + + void DoUpdateParameters() + { + } + + /* + * This function sets the generic parameters of a controller and runs the segmentation + */ + template<class TSegmenter> + void + SetGenericParametersAndRunSegmentation(const typename TSegmenter::ParamType params){ + + // Instantiate the controller + typedef typename lsgrm::Controller<TSegmenter> ControlerType; + typename ControlerType::Pointer controller = ControlerType::New(); + using GraphType = typename ControlerType::GraphType; + + // Set specific parameters + controller->SetSpecificParameters(params); + + // Set input image + ImageType::Pointer inputImage = GetParameterFloatVectorImage("in"); + controller->SetInputImage(inputImage); + controller->SetTemporaryFilesPrefix(GetParameterString("out")); + + // Set threshold + float thres = GetParameterFloat("threshold"); + controller->SetThreshold(thres*thres); + + // Switch tiling mode + int inputTilingMode = GetParameterInt("tiling"); + if (inputTilingMode == TILING_AUTO) + { + // Automatic mode + controller->SetTilingModeAuto(); + } + else if (inputTilingMode == TILING_USER) + { + // User mode + controller->SetTilingModeUser(); + controller->SetTileWidth(GetParameterInt("tiling.user.sizex")); + controller->SetTileHeight(GetParameterInt("tiling.user.sizey")); + controller->SetNumberOfFirstIterations(GetParameterInt("tiling.user.nfirstiter")); + } + else if (inputTilingMode == TILING_NONE) + { + // None mode + controller->SetTilingModeNone(); + } + else + { + otbAppLogFATAL("Unknown tiling mode!"); + } + + // Run the segmentation + controller->GetTilingLayout(); + + if (HasValue("xtileidx")) { + if (HasValue("ytileidx")) { + if (HasValue("threshold")) { + + controller->ProcessSingleTile(GetParameterInt("xtileidx"), + GetParameterInt("ytileidx"), + params); + + } + else { + otbAppLogFATAL("A tile is specified. Setting threshold parameter is mandatory."); + } + } + else { + otbAppLogFATAL("Only X index for tile is specified. Please set Y tile."); + } + } + else { + otbAppLogINFO("No tile specified. Only printing layout."); + } + + } + + void DoExecute() + { + + ImageType::Pointer inputImage = GetParameterFloatVectorImage("in"); + + // Switch criterion + int inputCriterion = GetParameterInt("criterion"); + if (inputCriterion == CRITERION_BAATZ) + { + grm::BaatzParam params; + params.m_SpectralWeight = GetParameterFloat("criterion.bs.cw"); + params.m_ShapeWeight = GetParameterFloat("criterion.bs.sw"); + SetGenericParametersAndRunSegmentation<BaatzSegmenterType>(params); + } + else if (inputCriterion == CRITERION_SPRING) + { + grm::SpringParam params; + SetGenericParametersAndRunSegmentation<SpringSegmenterType>(params); + } + else if (inputCriterion == CRITERION_FLS) + { + grm::FLSParam params; + SetGenericParametersAndRunSegmentation<FLSSegmenterType>(params); + } + else + { + otbAppLogFATAL("Unknow criterion!") + } + + } + +}; // app class +} // end of namespace wrapper +} // end of namespace otb + +OTB_APPLICATION_EXPORT(otb::Wrapper::SingleTileGRMGraph) diff --git a/include/lsgrmController.h b/include/lsgrmController.h index e538cbe..5be9e20 100644 --- a/include/lsgrmController.h +++ b/include/lsgrmController.h @@ -40,6 +40,10 @@ public: void Modified(); void RunSegmentation(); + void GetTilingLayout(); + void ProcessSingleTile(unsigned int xidx, unsigned int yidx, + const SegmentationParameterType& params); + void JustMergeAndAchieveSegmentation(const unsigned int maxiter); void SetSpecificParameters(const SegmentationParameterType& params); void SetInputImage(ImageType * inputImage); diff --git a/include/lsgrmController.txx b/include/lsgrmController.txx index aa645f7..f4f92fc 100644 --- a/include/lsgrmController.txx +++ b/include/lsgrmController.txx @@ -208,6 +208,117 @@ void Controller<TSegmenter>::RunSegmentation() } +/* + * Run the segmentation + * TODO: compute the correct number of iterations ! + */ +template<class TSegmenter> +void Controller<TSegmenter>::GetTilingLayout() +{ + itkDebugMacro(<< "Entering GetTilingLayout()"); + + CheckMemorySize(); + + if (m_TilingMode == LSGRM_TILING_AUTO || m_TilingMode == LSGRM_TILING_USER) + { + if(m_TilingMode == LSGRM_TILING_AUTO) + { + this->GetAutomaticConfiguration(); + } + else // m_TilingMode is LSGRM_TILING_USER + { + m_NbTilesX = std::floor(m_InputImage->GetLargestPossibleRegion().GetSize()[0] / m_TileWidth); + m_NbTilesY = std::floor(m_InputImage->GetLargestPossibleRegion().GetSize()[1] / m_TileHeight); + m_Margin = static_cast<unsigned int>(pow(2, m_NumberOfFirstIterations + 1) - 2); + } + + std::cout << + "--- Configuration: " << + "\n\tAvailable RAM: " << m_Memory << + "\n\tInput image dimensions: " << m_InputImage->GetLargestPossibleRegion().GetSize() << + "\n\tNumber of first iterations: " << m_NumberOfFirstIterations << + "\n\tStability margin: " << m_Margin << + "\n\tRegular tile size: " << m_TileWidth << " x " << m_TileHeight << + "\n\tTiling layout: " << m_NbTilesX << " x " << m_NbTilesY << std::endl; + + // Compute the splitting scheme + m_Tiles = SplitOTBImage<ImageType>(m_InputImage, m_TileWidth, m_TileHeight, m_Margin, + m_NbTilesX, m_NbTilesY, m_TemporaryFilesPrefix); + } +} + +template<class TSegmenter> +void Controller<TSegmenter>::ProcessSingleTile(unsigned int xidx, unsigned int yidx, + const SegmentationParameterType& params) +{ + const unsigned int imageWidth = m_InputImage->GetLargestPossibleRegion().GetSize()[0]; + const unsigned int imageHeight = m_InputImage->GetLargestPossibleRegion().GetSize()[1]; + + ProcessingTile currentTile = m_Tiles[yidx*m_NbTilesX+xidx]; + std::cout << "Processing tile " << (yidx*m_NbTilesX + xidx) << " / " << (m_NbTilesX*m_NbTilesY) << + " (" << yidx << ", " << xidx << ")" << + " start: [" << currentTile.region.GetIndex()[0] << ", " << currentTile.region.GetIndex()[1] << + "] size: [" << currentTile.region.GetSize()[0] << ", " << currentTile.region.GetSize()[1] << "]" << std::endl; + + typename ImageType::Pointer imageTile = ReadImageRegion<TSegmenter>(m_InputImage, currentTile.region); + // Segmenting image + std::cout << "\tSegmenting"; + + TSegmenter segmenter; + segmenter.SetParam(params); + segmenter.SetThreshold(m_Threshold); + segmenter.SetDoFastSegmentation(false); + segmenter.SetNumberOfIterations(m_NumberOfFirstIterations); + segmenter.SetInput(imageTile); + segmenter.Update(); + + // Rescale the graph to be in the reference of the image + std::cout << "\tRescaling graph..." << std::endl; + RescaleGraph<TSegmenter>(segmenter.m_Graph, + currentTile, + yidx, + xidx, + m_TileWidth, + m_TileHeight, + imageWidth); + + // Remove unstable segments + std::cout << "\tRemoving unstable segments..." << std::endl; + lsgrm::RemoveUnstableSegments<TSegmenter>(segmenter.m_Graph, currentTile, imageWidth); + + // Write graph to temporay directory + std::cout << "\tWriting graph..." << std::endl; + lsgrm::WriteGraph<TSegmenter>(segmenter.m_Graph, currentTile.nodeFileName, currentTile.edgeFileName); + + // Extract stability margin for all borders different from 0 imageWidth-1 and imageHeight -1 + // and write them to the stability margin + std::cout << "\tComputing stability margin..." << std::endl; + std::unordered_map<typename TSegmenter::NodePointerType, unsigned int> borderNodeMap; + + lsgrm::DetectBorderNodes<TSegmenter>(segmenter.m_Graph, currentTile, + borderNodeMap, imageWidth, imageHeight); + + lsgrm::ExtractStabilityMargin<TSegmenter>(borderNodeMap, 14); // in LSGRM numberOfNeighborLayers is pow(2, 3 + 1) - 2 + + lsgrm::WriteStabilityMargin<TSegmenter>(borderNodeMap, currentTile.nodeMarginFileName, currentTile.edgeMarginFileName); +} + +template<class TSegmenter> +void Controller<TSegmenter>::JustMergeAndAchieveSegmentation(const unsigned int maxiter) +{ + // Merge all the graphs + m_OutputGraph = MergeAllGraphsAndAchieveSegmentation<TSegmenter>( + m_SpecificParameters, + m_Threshold, + m_Tiles, + m_NbTilesX, + m_NbTilesY, + m_InputImage->GetLargestPossibleRegion().GetSize()[0], + m_InputImage->GetLargestPossibleRegion().GetSize()[1], + m_InputImage->GetNumberOfComponentsPerPixel(), + maxiter); +} + /* * Compute the memory occupied by one node * TODO: compute the exact value, e.g. on a given UNIX system, -- GitLab