Commit b6625040 authored by Gaetano Raffaele's avatar Gaetano Raffaele
Browse files

ENH: simplified resume option to recover interrupted segmentation.

No related merge requests found
Showing with 68 additions and 42 deletions
+68 -42
...@@ -116,13 +116,7 @@ private: ...@@ -116,13 +116,7 @@ private:
AddParameter(ParameterType_Bool, "resume", "Activate resume mode"); AddParameter(ParameterType_Bool, "resume", "Activate resume mode");
SetDefaultParameterInt("resume", 0); SetDefaultParameterInt("resume", 0);
AddParameter(ParameterType_Int, "resumetilerow", "Starting tile row in resume mode");
AddParameter(ParameterType_Int, "resumetilecol", "Starting tile column in resume mode");
SetDefaultParameterInt("resumetilerow", 0);
SetDefaultParameterInt("resumetilecol", 0);
MandatoryOff("resume"); MandatoryOff("resume");
MandatoryOff("resumetilerow");
MandatoryOff("resumetilecol");
} }
void DoUpdateParameters() void DoUpdateParameters()
...@@ -237,7 +231,7 @@ private: ...@@ -237,7 +231,7 @@ private:
// Resume mode? // Resume mode?
if (GetParameterInt("resume")) if (GetParameterInt("resume"))
{ {
controller->SetResumingMode(GetParameterInt("resumetilerow"),GetParameterInt("resumetilecol")); controller->SetResumingMode();
} }
// Run the segmentation // Run the segmentation
......
...@@ -45,7 +45,7 @@ public: ...@@ -45,7 +45,7 @@ public:
void SetInputImage(ImageType * inputImage); void SetInputImage(ImageType * inputImage);
void SetInternalMemoryAvailable(long long unsigned int v); // expecting a value in Mbytes. void SetInternalMemoryAvailable(long long unsigned int v); // expecting a value in Mbytes.
void SetResumingMode(unsigned int rRow, unsigned int rCol); void SetResumingMode();
void StopResumingMode(); void StopResumingMode();
/* Accessors */ /* Accessors */
...@@ -118,9 +118,6 @@ private: ...@@ -118,9 +118,6 @@ private:
unsigned int m_TileWidth; // regular tile width (i.e. not left tiles) unsigned int m_TileWidth; // regular tile width (i.e. not left tiles)
unsigned int m_TileHeight; // regular tile height (i.e. not bottom tiles) unsigned int m_TileHeight; // regular tile height (i.e. not bottom tiles)
bool m_Resuming; // True if in resuming mode bool m_Resuming; // True if in resuming mode
unsigned int m_ResumeTileRow; // X index of starting tile in resume mode
unsigned int m_ResumeTileCol; // Y index of starting tile in resume mode
/* read-only variables */ /* read-only variables */
LSGRMTilingMode m_TilingMode; // tiling mode (none/user/auto) LSGRMTilingMode m_TilingMode; // tiling mode (none/user/auto)
......
...@@ -19,8 +19,6 @@ Controller<TSegmenter>::Controller() ...@@ -19,8 +19,6 @@ Controller<TSegmenter>::Controller()
m_Threshold = 75; m_Threshold = 75;
m_Memory = 0; m_Memory = 0;
m_Resuming = false; m_Resuming = false;
m_ResumeTileRow = 0;
m_ResumeTileCol = 0;
} }
...@@ -94,6 +92,7 @@ void Controller<TSegmenter>::RunSegmentation() ...@@ -94,6 +92,7 @@ void Controller<TSegmenter>::RunSegmentation()
// temp. patch, maybe calculate real current memory after resuming graphs. // temp. patch, maybe calculate real current memory after resuming graphs.
long long unsigned int accumulatedMemory = 2 * m_Memory; long long unsigned int accumulatedMemory = 2 * m_Memory;
unsigned int nextTile = m_NbTilesX*m_NbTilesY;
accumulatedMemory = RunFirstPartialSegmentation<TSegmenter>( accumulatedMemory = RunFirstPartialSegmentation<TSegmenter>(
m_InputImage, m_InputImage,
...@@ -107,8 +106,9 @@ void Controller<TSegmenter>::RunSegmentation() ...@@ -107,8 +106,9 @@ void Controller<TSegmenter>::RunSegmentation()
m_TileWidth, m_TileWidth,
m_TileHeight, m_TileHeight,
isFusion, isFusion,
m_Resuming); m_Resuming,
nextTile);
#ifdef OTB_USE_MPI #ifdef OTB_USE_MPI
GatherUsefulVariables(accumulatedMemory, isFusion); GatherUsefulVariables(accumulatedMemory, isFusion);
#endif #endif
...@@ -116,7 +116,16 @@ void Controller<TSegmenter>::RunSegmentation() ...@@ -116,7 +116,16 @@ void Controller<TSegmenter>::RunSegmentation()
// Time monitoring // Time monitoring
ShowTime(t); ShowTime(t);
while(accumulatedMemory > m_Memory && isFusion) if (m_Resuming & nextTile < m_NbTilesX*m_NbTilesY) {
std::cout << "Detected resumable process from tile #" << nextTile << std::endl;
std::cout << "Forcing regular pass on the whole graph to update stability margins." << std::endl;
}
else {
std::cout << "Process resumed at first stage." << std::endl;
StopResumingMode();
}
while(m_Resuming || (accumulatedMemory > m_Memory && isFusion))
{ {
isFusion = false; isFusion = false;
accumulatedMemory = RunPartialSegmentation<TSegmenter>( accumulatedMemory = RunPartialSegmentation<TSegmenter>(
...@@ -131,8 +140,7 @@ void Controller<TSegmenter>::RunSegmentation() ...@@ -131,8 +140,7 @@ void Controller<TSegmenter>::RunSegmentation()
m_InputImage->GetNumberOfComponentsPerPixel(), m_InputImage->GetNumberOfComponentsPerPixel(),
isFusion, isFusion,
m_Resuming, m_Resuming,
m_ResumeTileRow, nextTile);
m_ResumeTileCol);
if (m_Resuming) StopResumingMode(); if (m_Resuming) StopResumingMode();
...@@ -474,19 +482,15 @@ std::vector<std::string> Controller<TSegmenter>::GetTemporaryFilesList() ...@@ -474,19 +482,15 @@ std::vector<std::string> Controller<TSegmenter>::GetTemporaryFilesList()
} }
template<class TSegmenter> template<class TSegmenter>
void Controller<TSegmenter>::SetResumingMode(unsigned int rRow, unsigned int rCol) void Controller<TSegmenter>::SetResumingMode()
{ {
m_Resuming = true; m_Resuming = true;
m_ResumeTileRow = rRow;
m_ResumeTileCol = rCol;
} }
template<class TSegmenter> template<class TSegmenter>
void Controller<TSegmenter>::StopResumingMode() void Controller<TSegmenter>::StopResumingMode()
{ {
m_Resuming = false; m_Resuming = false;
m_ResumeTileRow = 0;
m_ResumeTileCol = 0;
} }
} // end of namespace lsgrm } // end of namespace lsgrm
......
...@@ -55,7 +55,8 @@ long long unsigned int RunFirstPartialSegmentation( ...@@ -55,7 +55,8 @@ long long unsigned int RunFirstPartialSegmentation(
const unsigned int nbTilesY, const unsigned int nbTilesY,
const unsigned int tileWidth, const unsigned int tileWidth,
const unsigned int tileHeight, const unsigned int tileHeight,
bool& isFusion); bool& isFusion,
unsigned int& nextTile);
template<class TSegmenter> template<class TSegmenter>
long long unsigned int RunPartialSegmentation( long long unsigned int RunPartialSegmentation(
...@@ -69,9 +70,8 @@ long long unsigned int RunPartialSegmentation( ...@@ -69,9 +70,8 @@ long long unsigned int RunPartialSegmentation(
const unsigned int imageHeight, const unsigned int imageHeight,
const unsigned int imageBands, const unsigned int imageBands,
bool& isFusion, bool& isFusion,
bool resume = false, bool resume,
unsigned int rRow = 0, unsigned int& nextTile);
unsigned int rCol = 0);
template<class TSegmenter> template<class TSegmenter>
void RemoveUselessNodes(ProcessingTile& tile, void RemoveUselessNodes(ProcessingTile& tile,
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
//#include <unistd.h> //#include <unistd.h>
#include <fstream> #include <fstream>
#include <cstdio> #include <cstdio>
#include <sys/stat.h>
namespace lsgrm namespace lsgrm
{ {
...@@ -14,6 +15,13 @@ bool file_exists(const std::string& fileName) ...@@ -14,6 +15,13 @@ bool file_exists(const std::string& fileName)
return infile.good(); return infile.good();
} }
time_t last_mod_time(const std::string& fileName)
{
struct stat result;
if (!stat(fileName.c_str(), &result)) return result.st_mtime;
else return -1;
}
template<class TSegmenter> template<class TSegmenter>
typename TSegmenter::ImageType::Pointer ReadImageRegion( typename TSegmenter::ImageType::Pointer ReadImageRegion(
typename TSegmenter::ImageType * inputPtr, typename TSegmenter::ImageType * inputPtr,
...@@ -147,8 +155,7 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy ...@@ -147,8 +155,7 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy
const unsigned int imageBands, const unsigned int imageBands,
bool& isFusion, bool& isFusion,
bool resume, bool resume,
unsigned int rRow, unsigned int& nextTile)
unsigned int rCol)
{ {
long long unsigned int accumulatedMemory = 0; long long unsigned int accumulatedMemory = 0;
isFusion = false; isFusion = false;
...@@ -167,18 +174,19 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy ...@@ -167,18 +174,19 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy
{ {
// Get the current tile // Get the current tile
std::cout << "Processing tile " << row << ", " << col << std::endl; std::cout << "Processing tile " << row << ", " << col << std::endl;
ProcessingTile currentTile = tiles[row*nbTilesX + col]; unsigned int currTile = row*nbTilesX + col;
ProcessingTile currentTile = tiles[currTile];
// If resume mode, get accumulated memory on previous tiles // If resume mode, get accumulated memory on previous tiles
if (resume && (row*nbTilesX+col) < (rRow*nbTilesX+rCol) ) { if (resume && currTile < nextTile ) {
// Load the graph // Load the graph
std::cout << "\tResuming graph..." << std::endl; std::cout << "\tResuming graph..." << std::endl;
TSegmenter segmenter; TSegmenter segmenter;
ReadGraph<TSegmenter>(segmenter.m_Graph, currentTile.nodeFileName, currentTile.edgeFileName); ReadGraph<TSegmenter>(segmenter.m_Graph, currentTile.nodeFileName, currentTile.edgeFileName);
// Retrieve the amount of memory to store this graph // Retrieve the amount of memory to store this graph
std::cout << "\tGet graph memory..." << std::endl; std::cout << "\tGet graph memory..." << std::endl;
accumulatedMemory += segmenter.GetGraphMemory(); accumulatedMemory += segmenter.GetGraphMemory();
continue; continue;
} }
// Load the graph // Load the graph
...@@ -237,9 +245,13 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy ...@@ -237,9 +245,13 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy
// Write graph to temporay directory // Write graph to temporay directory
std::cout << "\tWrite graph..." << std::endl; std::cout << "\tWrite graph..." << std::endl;
WriteGraph<TSegmenter>(segmenter.m_Graph, currentTile.nodeFileName, currentTile.edgeFileName); WriteGraph<TSegmenter>(segmenter.m_Graph, currentTile.nodeFileName, currentTile.edgeFileName);
} }
} }
} }
if (resume)
nextTile = 0;
#ifdef OTB_USE_MPI #ifdef OTB_USE_MPI
otb::MPIConfig::Instance()->barrier(); otb::MPIConfig::Instance()->barrier();
...@@ -280,7 +292,7 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy ...@@ -280,7 +292,7 @@ long long unsigned int RunPartialSegmentation(const typename TSegmenter::ParamTy
} }
} }
std::cout << std::endl; std::cout << std::endl;
return accumulatedMemory; return accumulatedMemory;
} }
...@@ -648,7 +660,8 @@ long long unsigned int RunFirstPartialSegmentation( ...@@ -648,7 +660,8 @@ long long unsigned int RunFirstPartialSegmentation(
const unsigned int tileWidth, const unsigned int tileWidth,
const unsigned int tileHeight, const unsigned int tileHeight,
bool& isFusion, bool& isFusion,
bool resume) bool resume,
unsigned int& nextTile)
{ {
using ImageType = typename TSegmenter::ImageType; using ImageType = typename TSegmenter::ImageType;
...@@ -657,11 +670,14 @@ long long unsigned int RunFirstPartialSegmentation( ...@@ -657,11 +670,14 @@ long long unsigned int RunFirstPartialSegmentation(
long long unsigned int accumulatedMemory = 0; long long unsigned int accumulatedMemory = 0;
isFusion = false; isFusion = false;
bool accomplished = true;
const unsigned int numberOfNeighborLayers = static_cast<unsigned int>(pow(2, niter2 + 1) - 2); const unsigned int numberOfNeighborLayers = static_cast<unsigned int>(pow(2, niter2 + 1) - 2);
std::cout << "--- Running fist partial segmentation...\nNumber of neighbor layers " << numberOfNeighborLayers << std::endl; std::cout << "--- Running fist partial segmentation...\nNumber of neighbor layers " << numberOfNeighborLayers << std::endl;
time_t lastTime = -1;
for(unsigned int row = 0; row < nbTilesY; ++row) for(unsigned int row = 0; row < nbTilesY; ++row)
{ {
for(unsigned int col = 0; col < nbTilesX; col++) for(unsigned int col = 0; col < nbTilesX; col++)
...@@ -675,6 +691,11 @@ long long unsigned int RunFirstPartialSegmentation( ...@@ -675,6 +691,11 @@ long long unsigned int RunFirstPartialSegmentation(
if (resume && file_exists(currentTile.nodeFileName) && file_exists(currentTile.edgeFileName)) if (resume && file_exists(currentTile.nodeFileName) && file_exists(currentTile.edgeFileName))
{ {
int lastMod = last_mod_time(currentTile.nodeFileName);
if (lastMod > lastTime) {
lastTime = lastMod;
nextTile = row*nbTilesX + col + 1;
}
std::cout << "\tResuming graph..." << std::endl; std::cout << "\tResuming graph..." << std::endl;
TSegmenter segmenter; TSegmenter segmenter;
ReadGraph<TSegmenter>(segmenter.m_Graph, currentTile.nodeFileName, currentTile.edgeFileName); ReadGraph<TSegmenter>(segmenter.m_Graph, currentTile.nodeFileName, currentTile.edgeFileName);
...@@ -685,7 +706,9 @@ long long unsigned int RunFirstPartialSegmentation( ...@@ -685,7 +706,9 @@ long long unsigned int RunFirstPartialSegmentation(
isFusion = true; isFusion = true;
continue; continue;
} }
accomplished = false;
std::cout << "Processing tile " << (row*nbTilesX + col) << " / " << (nbTilesX*nbTilesY) << std::cout << "Processing tile " << (row*nbTilesX + col) << " / " << (nbTilesX*nbTilesY) <<
" (" << col << ", " << row << ")" << " (" << col << ", " << row << ")" <<
" start: [" << currentTile.region.GetIndex()[0] << ", " << currentTile.region.GetIndex()[1] << " start: [" << currentTile.region.GetIndex()[0] << ", " << currentTile.region.GetIndex()[1] <<
...@@ -740,10 +763,18 @@ long long unsigned int RunFirstPartialSegmentation( ...@@ -740,10 +763,18 @@ long long unsigned int RunFirstPartialSegmentation(
ExtractStabilityMargin<TSegmenter>(borderNodeMap, numberOfNeighborLayers); ExtractStabilityMargin<TSegmenter>(borderNodeMap, numberOfNeighborLayers);
WriteStabilityMargin<TSegmenter>(borderNodeMap, currentTile.nodeMarginFileName, currentTile.edgeMarginFileName); WriteStabilityMargin<TSegmenter>(borderNodeMap, currentTile.nodeMarginFileName, currentTile.edgeMarginFileName);
} }
} }
} // for each col } // for each col
} // for each row } // for each row
if (resume && !accomplished) {
std::cout << "\tSegmentation recovered during first partial segmentation." << std::endl;
nextTile = nbTilesX*nbTilesY;
}
if (resume && accomplished && nextTile == nbTilesX*nbTilesY)
nextTile = 0;
return accumulatedMemory; return accumulatedMemory;
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment