En raison du déménagement des baies serveurs, les services gitlab.irstea.fr et mattermost.irstea.fr seront interrompus le samedi 2 octobre 2021 au matin. Ils devraient revenir à la normale dans la journée.

Commit bdd9d960 authored by Rabotin Michael's avatar Rabotin Michael
Browse files

Add tests directory and author name

parent f30c7137
#!/bin/bash
############################################################################
#
# MODULE: hru-delin_all-steps.sh
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
# very simple, we just call all steps scripts with same arguments we got here
hru-delin_step1.sh "$@"
hru-delin_step2.sh "$@"
hru-delin_step3.sh "$@"
hru-delin_step4.sh "$@"
\ No newline at end of file
hru-delin_step4.sh "$@"
#!/bin/bash
############################################################################
#
# MODULE: hru-delin_step1.sh
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
# HRU-DELIN
# first step : hru-delin_init
......@@ -105,11 +120,20 @@ fi
# clean work environment
# ----------------------
# test id dir output exist
clean_files=`grep "files\s*:" $CONFIGFILE | cut -d ':' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
if [ -z "$clean_files" ]; then
clean_files=`grep "files\s*=" $CONFIGFILE | cut -d '=' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
fi
if [ -z "$clean_files" ]; then
echo "------------> ERROR : Output FILE Directory not provided !"
exit 1
fi
echo "clean files $clean_files"
# is the path absolute?
if [[ "$clean_files" = /* ]]; then
rm -rf $clean_files
......@@ -120,10 +144,23 @@ else
mkdir $CONFIGFILEPATH/$clean_files
fi
# test if mkdir works
if [ $? -ne 0 ] ; then
echo "------------> ERROR : Impossible to create Output directory !"
exit 1
fi
clean_results=`grep "results\s*:" $CONFIGFILE | cut -d ':' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
if [ -z "$clean_results" ]; then
clean_results=`grep "results\s*=" $CONFIGFILE | cut -d '=' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
fi
if [ -z "$clean_results" ]; then
echo "------------> ERROR : Output RESULTS Directory not provided !"
exit 1
fi
echo "clean results $clean_results"
# is the path absolute?
if [[ "$clean_results" = /* ]]; then
......
#!/bin/bash
############################################################################
#
# MODULE: hru-delin_step2.sh
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
# HRU-DELIN
# 2nd step : hru-delin_basins
......@@ -107,6 +121,12 @@ clean_files=`grep "files\s*:" $CONFIGFILE | cut -d ':' -f2 | sed -e 's/\s*$//' |
if [ -z "$clean_files" ]; then
clean_files=`grep "files\s*=" $CONFIGFILE | cut -d '=' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
fi
if [ -z "$clean_files" ]; then
echo "------------> ERROR : Output FILE Directory not provided !"
exit 1
fi
# is the path absolute?
if [[ "$clean_files" = /* ]]; then
rm -f $clean_files/step2* $clean_files/step3*
......@@ -118,6 +138,12 @@ clean_results=`grep "results\s*:" $CONFIGFILE | cut -d ':' -f2 | sed -e 's/\s*$/
if [ -z "$clean_results" ]; then
clean_results=`grep "results\s*=" $CONFIGFILE | cut -d '=' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
fi
if [ -z "$clean_results" ]; then
echo "------------> ERROR : Output RESULTS Directory not provided !"
exit 1
fi
# is the path absolute?
if [[ "$clean_results" = /* ]]; then
rm -rf $clean_results
......@@ -127,6 +153,12 @@ else
mkdir $CONFIGFILEPATH/$clean_results
fi
# test if mkdir works
if [ $? -ne 0 ] ; then
echo "------------> ERROR : Impossible to create Output directory !"
exit 1
fi
# -----------------------------------
# exec second step of HRU-DELIN batch
# -----------------------------------
......
#!/bin/bash
############################################################################
#
# MODULE: hru-delin_step3.sh
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
# HRU-DELIN
# 3rd step : hru-delin_hrugen
......@@ -108,6 +121,12 @@ clean_files=`grep "files\s*:" $CONFIGFILE | cut -d ':' -f2 | sed -e 's/\s*$//' |
if [ -z "$clean_files" ]; then
clean_files=`grep "files\s*=" $CONFIGFILE | cut -d '=' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
fi
if [ -z "$clean_files" ]; then
echo "------------> ERROR : Output FILE Directory not provided !"
exit 1
fi
# is the path absolute?
if [[ "$clean_files" = /* ]]; then
rm -f $clean_files/step3*
......@@ -118,6 +137,12 @@ clean_results=`grep "results\s*:" $CONFIGFILE | cut -d ':' -f2 | sed -e 's/\s*$/
if [ -z "$clean_results" ]; then
clean_results=`grep "results\s*=" $CONFIGFILE | cut -d '=' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
fi
if [ -z "$clean_results" ]; then
echo "------------> ERROR : Output RESULTS Directory not provided !"
exit 1
fi
# is the path absolute?
if [[ "$clean_results" = /* ]]; then
rm -rf $clean_results
......@@ -127,6 +152,12 @@ else
mkdir $CONFIGFILEPATH/$clean_results
fi
# test if mkdir works
if [ $? -ne 0 ] ; then
echo "------------> ERROR : Impossible to create Output directory !"
exit 1
fi
# -----------------------------------
# exec third step of HRU-DELIN batch
# -----------------------------------
......
#!/bin/bash
############################################################################
#
# MODULE: hru-delin_step4.sh
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
# HRU-DELIN
# 4th step : hru-delin_parms_J2000
......@@ -109,6 +123,12 @@ clean_results=`grep "results\s*:" $CONFIGFILE | cut -d ':' -f2 | sed -e 's/\s*$/
if [ -z "$clean_results" ]; then
clean_results=`grep "results\s*=" $CONFIGFILE | cut -d '=' -f2 | sed -e 's/\s*$//' | sed -e 's/^\s*//'`
fi
if [ -z "$clean_results" ]; then
echo "------------> ERROR : Output RESULTS Directory not provided !"
exit 1
fi
# is the path absolute?
if [[ "$clean_results" = /* ]]; then
rm -rf $clean_results
......@@ -118,6 +138,13 @@ else
mkdir $CONFIGFILEPATH/$clean_results
fi
# test if mkdir works
if [ $? -ne 0 ] ; then
echo "------------> ERROR : Impossible to create Output directory !"
exit 1
fi
rm -f topologie_* tmp/topologie_*
# -----------------------------------
......
#!/bin/bash
############################################################################
#
# MODULE: tools.sh
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
buildGrassEnv() {
envPath=$1
......@@ -48,4 +61,4 @@ t-b resol: 1
" > $gisdbase/$location/$mapset/WIND
cp $gisdbase/$location/$mapset/WIND $gisdbase/$location/$mapset/DEFAULT_WIND
}
\ No newline at end of file
}
############################################################################
#
# MODULE: awk.py
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
# coding: utf-8
# this file contains the converted old awk scripts which were not portable (to windows)
......
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
############################################################################
#
# MODULE: circleKill.py
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
import sys
try:
import dbf
......@@ -105,4 +122,4 @@ if __name__ == '__main__':
outPath = sys.argv[2]
circleKill(inPath, outPath)
else:
sys.exit('Please provide two arguments: INPUT_PATH OUTPUT_PATH_PREFIX')
\ No newline at end of file
sys.exit('Please provide two arguments: INPUT_PATH OUTPUT_PATH_PREFIX')
# coding: utf-8
############################################################################
#
# MODULE: grassUtils.py
# AUTHOR(S): Julien Veyssier
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
import os, sys, shutil
try:
# Python 3
......
# coding: utf-8
'''
MODULE: hru-delin_init
AUTHOR(S): adapted from GRASS-HRU (ILMS) - JENA University
by IRSTEA - Christine Barachet
############################################################################
#
# MODULE: hrudelin_1_init.py
# AUTHOR(S): adapted from GRASS-HRU (ILMS) - JENA University
# by IRSTEA - Christine Barachet,
# Julien Veyssier
# PURPOSE: Prepare files for the next steps of HRU Delineation
# shapefile : selected gauges
# rasters : bounded DEM and others (geology, soils, landuse)
# DEM, slope and aspect with values of reclassification
# calculates oriented flows and subbasins
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
PURPOSE: Prepare files for the next steps of HRU Delineation
shapefile : selected gauges
rasters : bounded DEM and others (geology, soils, landuse)
DEM, slope and aspect with values of reclassification
calculates oriented flows and subbasins
'''
# to keep python2 compatibility
from __future__ import print_function
......@@ -61,13 +70,37 @@ def get_polygon_bounds(shape, xmin_slect, ymax_slect, xmax_slect, ymin_slect):
max(ymin_slect, ymin)
)
def isfloat(value):
try:
float(value)
return True
except ValueError:
return False
def isint(value):
try:
int(value)
return True
except ValueError:
return False
def get_coords(parms, xmin_slect, ymax_slect, xmax_slect, ymin_slect):
if not isfloat(parms.get('surface', 'west')) or isfloat(parms.get('surface', 'north')) or isfloat(parms.get('surface', 'east')) or isfloat(parms.get('surface', 'south')):
sys.exit('------------> ERROR : coordinates for surface selection is not valid')
xmin_slect = max(xmin_slect, float(parms.get('surface', 'west')))
ymax_slect = min(ymax_slect, float(parms.get('surface', 'north')))
xmax_slect = min(xmax_slect, float(parms.get('surface', 'east')))
ymin_slect = max(ymin_slect, float(parms.get('surface', 'south')))
print("hello",xmin_slect)
return xmin_slect, ymax_slect, xmax_slect, ymin_slect
try:
# Python 3
from subprocess import DEVNULL
......@@ -75,12 +108,20 @@ except ImportError:
#DEVNULL = open('cbtrace_step1', 'wb')
DEVNULL = open(os.devnull, 'wb')
'''
MAIN
'''
def main(parms_file):
print('---------- HRU-delin Step 1 started ---------------------------------------------')
configFileDir = os.path.dirname(parms_file)
# create main env
buildGrassEnv(os.path.join(configFileDir, 'grass_db'), 'hru-delin')
......@@ -89,26 +130,81 @@ def main(parms_file):
parms = ConfigParser.ConfigParser(allow_no_value=True)
parms.read(parms_file)
directory = parms.get('dir_in', 'dir')
# manage absolute and relative paths
# Test parameters from configuration file
## test if directory is valid
if not os.path.isdir(directory):
sys.exit('------------> ERROR : In Directoy is not valid')
## manage absolute and relative paths
if not os.path.isabs(directory):
directory = os.path.join(configFileDir, directory)
directory_out = parms.get('dir_out', 'files')
if not os.path.isabs(directory_out):
directory_out = os.path.join(configFileDir, directory_out)
## Test if dem exist
dem = os.path.join(directory, str(parms.get('files_in', 'dem')))
if not os.path.isfile(dem):
sys.exit('------------> ERROR : Input Dem not found')
## Test if gauges exist
gauges = os.path.join(directory, str(parms.get('files_in', 'gauges')))
if not os.path.isfile(gauges):
sys.exit('------------> ERROR : Input Gauges not found')
## Test if gauges is point or multipoint
ds=ogr.Open(gauges)
layer=ds.GetLayer()
layer_defn=layer.GetLayerDefn()
if ogr.GeometryTypeToName(layer_defn.GetGeomType()) != 'Point':
sys.exit('------------> ERROR : Input Gauges is not Point Geometry (multi Point not allowed)')
## if hgeon, landuse and soil provided, test if exist
for data in parms.items('data'):
data = os.path.join(directory, data[1])
if not os.path.isfile(data):
print(data)
sys.exit('------------> ERROR : Input data not found' )
## if irrig_rast provided, test if is valid
if str(parms.get('irrigation', 'irrig_rast')) != '':
irr_rast_test = os.path.join(directory, str(parms.get('irrigation', 'irrig_rast')))
if not os.path.isfile(irr_rast_test):
sys.exit('------------> ERROR : Irrigation raster not found' )
## if rules_auto_dem is yes, test if step_dem valid
if (parms.get('reclass_dem', 'rules_auto_dem')) == 'yes':
if not isint(parms.get('reclass_dem', 'step_dem')):
sys.exit('------------> ERROR : Step dem value not provided or is not integer' )
## test if basin min size is valid
if not isint(parms.get('basin_min_size', 'size')):
sys.exit('------------> ERROR : Basin min size value not provided or is not integer' )
# Initializing the variables for cutting up of layers
xmin_slect = ymin_slect = 0.0
ymax_slect = xmax_slect = 9999999999.9
dem = os.path.join(directory, str(parms.get('files_in', 'dem')))
dem_name = os.path.basename(parms.get('files_in', 'dem')).split('.')[0]
selection = (parms.get('surface', 'selection'))
xmin_slect, ymax_slect, xmax_slect, ymin_slect = get_raster_bounds(dem, xmin_slect, ymax_slect, xmax_slect, ymin_slect)
## test if selection is valid
if selection == 'total':
pass
elif selection == 'polygon':
polygon_test = os.path.join(directory, str(parms.get('surface', 'polygon')))
if not os.path.isfile(polygon_test):
sys.exit('------------> ERROR : Polygon layer for surface selection not found')
xmin_slect, ymax_slect, xmax_slect, ymin_slect = get_polygon_bounds(os.path.join(directory, parms.get('surface', 'polygon')), xmin_slect, ymax_slect, xmax_slect, ymin_slect)
elif selection == 'coords':
xmin_slect, ymax_slect, xmax_slect, ymin_slect = get_coords(parms, xmin_slect, ymax_slect, xmax_slect, ymin_slect)
......@@ -174,15 +270,20 @@ def main(parms_file):
# DEM
dem_rcl = 'dem_rcl'
if (parms.get('reclass_dem', 'rules_auto_dem')) == 'yes':
pRecode = grass_feed_command('r.recode', input=dem_filled, output=dem_rcl, rules='-', quiet=True)
step_reclass = int(parms.get('reclass_dem', 'step_dem'))
alti_min = float(grass_parse_command('r.info', flags='r', map=dem_filled, stdout=DEVNULL, stderr=DEVNULL)['min'])
alti_max = float(grass_parse_command('r.info', flags='r', map=dem_filled, stdout=DEVNULL, stderr=DEVNULL)['max'])
# just a copy, still do the pipe
recl_dem_file = os.path.join(directory_out, 'reclass_rules_dem')
with open(recl_dem_file, 'w') as dem_rules:
new_max = math.floor(alti_max / step_reclass) * step_reclass + step_reclass
if alti_min < 0:
......@@ -208,7 +309,9 @@ def main(parms_file):
if not os.path.isfile(recl_dem_file):
sys.exit('------------> File reclass_rules_dem is missing')
# r.recode does a better job with grass7, values close to interval limits are now correctly reclassified
grass_run_command('r.recode', input=dem_filled, output=dem_rcl, rules=recl_dem_file, stdout=DEVNULL, stderr=DEVNULL)
# Save the rules to directory_out
recl_dem_name = os.path.basename(recl_dem_file)
shutil.copyfile(recl_dem_file, os.path.join(directory_out, recl_dem_name))
......@@ -219,6 +322,7 @@ def main(parms_file):
if (parms.get('reclass_slope', 'rules_auto_slope')) == 'yes':
pRecode = grass_feed_command('r.recode', input=dem_slope, output=slp_rcl, rules='-', quiet=True)
recl_slope_file = os.path.join(directory_out, 'reclass_rules_slope')
default_recl_slope_file = os.path.join(DATA_PATH, 'reclass_default_rules_slope')
slp_min = float(grass_parse_command('r.info', flags='r', map=dem_slope, stdout=DEVNULL, stderr=DEVNULL)['min'])
......
# coding: utf-8
"""
MODULE: hru-delin_basins
AUTHOR(S): adapted from GRASS-HRU (ILMS) - JENA University
by IRSTEA - Christine Barachet
PURPOSE: 1. Relocates the gauges on the reaches
2. Calculates watersheds at the gauges
############################################################################
#
# MODULE: hru-delin_basins.py
# AUTHOR(S): adapted from GRASS-HRU (ILMS) - JENA University
# by IRSTEA - Christine Barachet,
# Julien Veyssier
# PURPOSE: 1. Relocates the gauges on the reaches
# 2. Calculates watersheds at the gauges
#
#
# COPYRIGHT: (C) 2020 UR RIVERLY - INRAE
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file LICENSE that comes with
# HRU-DELIN for details.
#
#############################################################################
"""
# to keep python2 compatibility
from __future__ import print_function
......@@ -40,6 +52,15 @@ try:
except ImportError:
DEVNULL = open(os.devnull, 'wb')
def isint(value):
try:
int(value)
return True
except ValueError:
return False
def write_log(msg, buff, start, end, best_x, best_y):
horiz_shift = best_x - start - end
vertic_shift = best_y - start - end + 1
......@@ -79,6 +100,10 @@ def snapping_gauges_to_reaches(parms, directory_out):
dist_ok_2 = 0
else:
nb_trials = 2
if not isint(parms.get('auto_relocation', 'surface_tolerance_2')):
sys.exit('------------> ERROR : Surface_tolerance_2 is not integer' )
if not isint(parms.get('auto_relocation', 'distance_tolerance_2')):
sys.exit('------------> ERROR : Distance_tolerance_2 is not integer' )
surface_ok_2 = int(parms.get('auto_relocation', 'surface_tolerance_2'))
dist_ok_2 = int(parms.get('auto_relocation', 'distance_tolerance_2' ))
......@@ -126,9 +151,20 @@ def snapping_gauges_to_reaches(parms, directory_out):
for y in range(0, (max_dist * 2 + 1)):