Merge branch 'packaging_ci' into 'develop'

CI: packaging

See merge request orfeotoolbox/otb!491

.gitlab-ci.yml

@@ -44,6 +44,7 @@ fast-build:
     - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
     - ccache -s
 
+
 .common-build:
   extends: .general
   only: [merge_requests]
@@ -56,6 +57,12 @@ fast-build:
     - git checkout -f -q $CI_COMMIT_SHA
   after_script:
     - python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH $CI_MERGE_REQUEST_REF_PATH
+  artifacts:
+    when: on_failure
+    expire_in: 24 hrs
+    paths:
+      - log/*.txt
+      - build/*/*.log
 
 debian-build:
   extends: .common-build
@@ -91,30 +98,34 @@ ubuntu-llvm:
       # This recovers logs from superbuild build
       - build/*/*/*/*.log
 
+## Ubuntu superbuild
 ubuntu-superbuild-prepare:
   extends: .common-prepare
   image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
   script:
     - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
 
-centos-superbuild-prepare:
-  extends: .common-prepare
-  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
-  script:
-    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
-
 ubuntu-superbuild-build:
   extends: .common-build
   image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
   script:
     - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
+    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
   dependencies:
     - ubuntu-superbuild-prepare
 
+## Centos superbuild
+centos-superbuild-prepare:
+  extends: .common-prepare
+  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
+  script:
+    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
+
 centos-superbuild-build:
   extends: .common-build
   image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
   script:
     - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
+    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
   dependencies:
     - centos-superbuild-prepare

CI/configure_options.cmake

@@ -38,7 +38,7 @@ OTB_USE_GLEW:BOOL=ON
 OTB_USE_GLFW:BOOL=ON
 OTB_USE_GLUT:BOOL=ON
 OTB_USE_GSL:BOOL=ON
-OTB_USE_LIBKML:BOOL=ON
+OTB_USE_LIBKML:BOOL=OFF
 OTB_USE_LIBSVM:BOOL=ON
 OTB_USE_MPI:BOOL=OFF
 OTB_USE_MUPARSER:BOOL=ON

CI/main_packages.cmake

+#
+# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
+#
+# This file is part of Orfeo Toolbox
+#
+#     https://www.orfeo-toolbox.org/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We are included from main_superbuild.cmake
+# This script is a prototype for the future CI, it may evolve rapidly in a near future
+
+include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
+
+set (ENV{LANG} "C") # Only ascii output
+
+get_filename_component( OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY )
+
+set ( DEBUG "1" )
+
+set ( CTEST_BUILD_CONFIGURATION "Release" )
+set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
+set ( CTEST_BUILD_FLAGS "-j1" )
+set ( CTEST_BUILD_NAME "Packages" )
+set ( CTEST_SITE "${IMAGE_NAME}" )
+
+# Find the build name and CI profile
+set_dash_build_name()
+
+# Directory variable
+set ( CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}/Packaging" )
+set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build_packages/" )
+set ( CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/install_packages" )
+set ( PROJECT_SOURCE_DIR "${CTEST_SOURCE_DIRECTORY}" )
+
+# Copy back xdk for RUN_PATH reason
+# We might want to change the name of artifact in main_superbuild
+# file ( COPY "${OTB_SOURCE_DIR}/install/xdk" DESTINATION "${CI_ROOT_DIR}")
+# Packages case:
+# SUPERBUILD_BINARY_DIR this is needed for OTB_BINARY_DIR, not sure we need it
+# SUPERBUILD_INSTALL_DIR do we need it? it seems so... We will set it to anything
+# DOWNLOAD_LOCATION
+# OTB_BINARY_DIR
+# CMAKE_INSTALL_PREFIX
+set ( CONFIGURE_OPTIONS
+"-DCMAKE_INSTALL_PREFIX=${CTEST_INSTALL_DIRECTORY};\
+-DOTB_BINARY_DIR=${OTB_SOURCE_DIR}/build;\
+-DSUPERBUILD_INSTALL_DIR=${OTB_SOURCE_DIR}/xdk;\
+-DSUPERBUILD_BINARY_DIR=${OTB_SOURCE_DIR}/build;" )
+
+# Look for a GIT command-line client.
+find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
+
+# Sources are already checked out : do nothing for update
+set(CTEST_GIT_UPDATE_CUSTOM echo No update)
+
+
+ctest_start( Experimental TRACK CI_Package )
+
+ctest_update( SOURCE "${OTB_SOURCE_DIR}" )
+
+
+ctest_configure(
+  BUILD "${CTEST_BINARY_DIRECTORY}"
+  SOURCE "${CTEST_SOURCE_DIRECTORY}"
+  OPTIONS "${CONFIGURE_OPTIONS}"
+  RETURN_VALUE _configure_rv
+  CAPTURE_CMAKE_ERROR _configure_error
+  )
+
+if( NOT _configure_rv EQUAL 0 )
+  ctest_submit()
+  message( SEND_ERROR "An error occurs during ctest_configure.")
+  return()
+endif()
+
+
+ctest_build(
+  BUILD "${CTEST_BINARY_DIRECTORY}"
+  TARGET PACKAGE-OTB
+  RETURN_VALUE _build_rv
+  CAPTURE_CMAKE_ERROR _build_error
+  )
+
+if( NOT _build_rv EQUAL 0 )
+  message( SEND_ERROR "An error occurs during ctest_build.")
+endif()
+
+
+ctest_test(
+  PARALLEL_LEVEL 1
+  RETURN_VALUE _test_rv
+  CAPTURE_CMAKE_ERROR _test_error
+  )
+
+if( NOT _test_rv EQUAL 0 )
+  message( SEND_ERROR "An error occurs during ctest_test.")
+endif()
+
+ctest_submit()

CI/main_superbuild.cmake

@@ -24,6 +24,7 @@ include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
 set (ENV{LANG} "C") # Only ascii output
 get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
 
+set ( DEBUG "1" )
 ###########################################################################
 ###########################################################################
 # Download xkd
@@ -76,9 +77,19 @@ set (CMAKE_COMMAND "cmake")
 execute_process(
   COMMAND ${CMAKE_COMMAND} "-E" "tar" "xf"
   "${OTB_SOURCE_DIR}/superbuild-artifact/SuperBuild_Install.tar"
+  RESULT_VARIABLE tar_res
+  OUTPUT_VARIABLE tar_out
+  ERROR_VARIABLE tar_err
   WORKING_DIRECTORY ${OTB_SOURCE_DIR}
   )
 
+if ( DEBUG )
+  message( "${CMAKE_COMMAND} -E tar xf ${OTB_SOURCE_DIR}/superbuild-artifact/SuperBuild_Install.tar")
+  message( "tar_res: '${clone_res}'" )
+  message( "tar_out: '${tar_out}'" )
+  message( "tar_err: '${tar_err}'" )
+endif()
+
 set( XDK_PATH "${OTB_SOURCE_DIR}/xdk")
 
 if ( DEBUG )
@@ -107,19 +118,24 @@ set_dash_build_name()
 # Directory variable
 set ( CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}" )
 set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" )
-set ( CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/install/" )
+# install in same directory as xdk to be able to produce
+# package afterwards
+set ( CTEST_INSTALL_DIRECTORY "${XDK_PATH}" )
 set ( PROJECT_SOURCE_DIR "${OTB_SOURCE_DIR}" )
 
 set (CONFIGURE_OPTIONS  "")
 include ( "${CMAKE_CURRENT_LIST_DIR}/configure_options.cmake" )
+
+# For superbuild we need remote module 
+foreach(remote_module DiapOTBModule OTBTemporalGapFilling)#otbGRM #Mosaic # #SertitObject
+    set ( CONFIGURE_OPTIONS 
+      "${CONFIGURE_OPTIONS}-DModule_${remote_module}:BOOL=ON;")
+endforeach()
+
 # SuperBuild case : one more configure option
 set ( CONFIGURE_OPTIONS
   "${CONFIGURE_OPTIONS}-DCMAKE_PREFIX_PATH=${XDK_PATH};")
 
-# Hack because there is no more superbuild available (LIBKML)
-set ( CONFIGURE_OPTIONS
-  "${CONFIGURE_OPTIONS}-DOTB_USE_LIBKML:BOOL=OFF;" )
-
 # FIX ME this part might platform dependent
 set( GDAL_DATA "${XDK_PATH}/share/gdal" )
 set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" )
@@ -145,6 +161,11 @@ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
     RETURN_VALUE _configure_rv
     CAPTURE_CMAKE_ERROR _configure_error
     )
+# Configure log
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/configure_return_value_log.txt" "${_configure_rv}")
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/configure_cmake_error_log.txt" "${_configure_error}")
 
 if ( NOT _configure_rv EQUAL 0 )
   ctest_submit()
@@ -156,6 +177,14 @@ ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
             CAPTURE_CMAKE_ERROR _build_error
             )
 
+# Build log
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/build_return_value_log.txt" "${_build_rv}")
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/build_cmake_error_log.txt" "${_build_error}")
+
+# SEND_ERROR if build error
+# FATAL_ERROR if build error?
 if ( NOT _build_rv EQUAL 0 )
   message( SEND_ERROR "An error occurs during ctest_build.")
 endif()
@@ -166,8 +195,41 @@ ctest_test(PARALLEL_LEVEL 8
            CAPTURE_CMAKE_ERROR _test_error
            )
 
+# Test log
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/test_return_value_log.txt" "${_test_rv}")
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/test_cmake_error_log.txt" "${_test_error}")
+
 if ( NOT _test_rv EQUAL 0 )
   message( WARNING "Some tests have failed.")
 endif()
 
 ctest_submit()
+
+# We need to install OTB for package purposes
+set ( MAKE_COMMAND "make")
+execute_process(
+  COMMAND ${MAKE_COMMAND} "install"
+  WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}
+  RESULT_VARIABLE install_rv
+  OUTPUT_VARIABLE install_out
+  ERROR_VARIABLE install_err
+  )
+
+if ( DEBUG )
+  message( "Install output")
+  message( "install_res = ${install_res}" )
+  message( "install_out = ${install_out}" )
+  message( "install_err = ${install_err}" )
+endif()
+
+# Install log
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/install_out_log.txt" "${install_out}")
+file ( WRITE 
+  "${OTB_SOURCE_DIR}/log/install_error_log.txt" "${install_err}")
+
+if ( NOT install_rv EQUAL 0 )
+  message( SEND_ERROR "Install have failed.")
+endif()

CI/prepare_superbuild.cmake

@@ -152,7 +152,6 @@ if ( DEBUG )
 endif()
 
 if ( ( NOT ${_build_nb_err} EQUAL 0 ) OR ( ${_build_error} EQUAL -1 ))
-  ctest_submit()
   message( FATAL_ERROR "An error occurs during ctest_build.")
 endif()
 

Data/Input/SP67_FR_subset_1_cutline.prj

+PROJCS["RGF93_Lambert_93",GEOGCS["GCS_RGF93",DATUM["D_RGF_1993",SPHEROID["GRS_1980",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Lambert_Conformal_Conic"],PARAMETER["standard_parallel_1",49],PARAMETER["standard_parallel_2",44],PARAMETER["latitude_of_origin",46.5],PARAMETER["central_meridian",3],PARAMETER["false_easting",700000],PARAMETER["false_northing",6600000],UNIT["Meter",1]]
\ No newline at end of file

Data/Input/SP67_FR_subset_1_cutline.qpj

+PROJCS["RGF93 / Lambert-93",GEOGCS["RGF93",DATUM["Reseau_Geodesique_Francais_1993",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6171"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4171"]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["standard_parallel_1",49],PARAMETER["standard_parallel_2",44],PARAMETER["latitude_of_origin",46.5],PARAMETER["central_meridian",3],PARAMETER["false_easting",700000],PARAMETER["false_northing",6600000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH],AUTHORITY["EPSG","2154"]]