diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9525bdc2898a29d2e111dba868cb80dc60bc912f..f59baac2626b15469be898b1003c153933de6885 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ # But Git fails to manage efficiently numerous LFS data. # We have to use directly git-lfs wrapping commands to have an # efficient cloning step. -# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of +# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of # GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318) variables: @@ -53,17 +53,19 @@ fast-build: - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast - ccache -s -debian-build: +.common-build: extends: .general only: [merge_requests] stage: build + +debian-build: + extends: .common-build image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable script: - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc -.common-build: - extends: .general - stage: build +ubuntu-llvm: + extends: .common-build image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:18.04 script: - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm @@ -73,15 +75,10 @@ debian-build: - build/Documentation/Cookbook/latex/CookBook-*.pdf - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2 -ubuntu-llvm: - only: [merge_requests] - extends: .common-build - -superbuild-prepare: - only: [merge_requests] +.common-prepare: extends: .general + only: [merge_requests] stage: prepare - image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04 before_script: # This override the previous before_script - git checkout $CI_COMMIT_REF_NAME @@ -95,8 +92,6 @@ superbuild-prepare: - ssh-add <(echo "$K8S_SECRET_SSH") # This is for debug, we are checking the owner of the ssh key - ssh -o StrictHostKeyChecking=no -T git@gitlab.orfeo-toolbox.org - script: - - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base after_script: - echo "Nothing to do for after_script" artifacts: @@ -106,12 +101,30 @@ superbuild-prepare: - build/*/*/*/*.log - sb_branch.txt -superbuild-build: - only: [merge_requests] - extends: .general - stage: build +ubuntu-superbuild-prepare: + extends: .common-prepare image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04 script: - - ctest -VV -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base + - ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base + +centos-superbuild-prepare: + extends: .common-prepare + image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6 + script: + - ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base + +ubuntu-superbuild-build: + extends: .common-build + image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04 + script: + - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base + dependencies: + - ubuntu-superbuild-prepare + +centos-superbuild-build: + extends: .common-build + image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6 + script: + - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base dependencies: - - superbuild-prepare + - centos-superbuild-prepare diff --git a/CI/configure_option.cmake b/CI/configure_option.cmake index 26502b11bfa758e483b8b60cafb62e1ad6ff9067..d0b41c6adc0fe7546fecfbb698a5710cf849755d 100644 --- a/CI/configure_option.cmake +++ b/CI/configure_option.cmake @@ -19,13 +19,13 @@ # # This script is a prototype for the future CI, it may evolve rapidly in a near future -#This file set the following variable : +#This file set the following variable : # * otb_build_project_option # * otb_use_option # * otb_wrap_option # * CONFIGURE_OPTIONS -set (otb_build_project_option +set (otb_build_project_option "BUILD_COOKBOOK:BOOL=OFF BUILD_EXAMPLES:BOOL=ON BUILD_SHARED_LIBS:BOOL=ON @@ -40,7 +40,7 @@ OTB_USE_GLUT:BOOL=ON OTB_USE_GSL:BOOL=ON OTB_USE_LIBKML:BOOL=ON OTB_USE_LIBSVM:BOOL=ON -OTB_USE_MPI:BOOL=ON +OTB_USE_MPI:BOOL=OFF OTB_USE_MUPARSER:BOOL=ON OTB_USE_MUPARSERX:BOOL=ON OTB_USE_OPENCV:BOOL=ON diff --git a/CI/macros.cmake b/CI/macros.cmake new file mode 100644 index 0000000000000000000000000000000000000000..544daca845fd1b0d8fe36d746cd698fa09ce3a76 --- /dev/null +++ b/CI/macros.cmake @@ -0,0 +1,53 @@ +# +# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES) +# +# This file is part of Orfeo Toolbox +# +# https://www.orfeo-toolbox.org/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is for the superbuild build on the CI platform + +# Find the build name and CI profile +macro( set_dash_build_name ) + set(ci_profile wip) + + set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}") + set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}") + set(ci_mr_iid "$ENV{CI_MERGE_REQUEST_IID}") + + set(ci_pipeline_id "$ENV{CI_PIPELINE_ID}") + set(ci_job_id "$ENV{CI_JOB_ID}") + + set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}") + + set (CTEST_BUILD_NAME ${ci_short_sha}) + + if(ci_mr_source AND ci_mr_target AND ci_mr_iid) + set (CTEST_BUILD_NAME "${ci_mr_source} (MR ${ci_mr_iid}) P-${ci_pipeline_id}") + set(ci_profile mr) + + elseif(ci_ref_name) + set (CTEST_BUILD_NAME "${ci_ref_name} P-${ci_pipeline_id}") + + if("${ci_ref_name}" STREQUAL "develop") + set(ci_profile develop) + + elseif("${ci_ref_name}" MATCHES "^release-[0-9]+\\.[0-9]+\$") + set(ci_profile release) + endif() + + endif() + +endmacro() diff --git a/CI/main_ci.cmake b/CI/main_ci.cmake index c0a1cf37f37e88b86af2a88efab623ff02079b0b..eba3829c5017383a253e86a2d197ec0f1b860f1a 100644 --- a/CI/main_ci.cmake +++ b/CI/main_ci.cmake @@ -18,6 +18,8 @@ # limitations under the License. # +include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" ) + # This script is a prototype for the future CI, it may evolve rapidly in a near future get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY) set (ENV{LANG} "C") # Only ascii output @@ -36,23 +38,7 @@ else() endif() # Find the build name and CI profile -set(ci_profile wip) -set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}") -set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}") -set(ci_mr_iid "$ENV{CI_MERGE_REQUEST_IID}") -set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}") -set (CTEST_BUILD_NAME ${ci_short_sha}) -if(ci_mr_source AND ci_mr_target AND ci_mr_iid) - set (CTEST_BUILD_NAME "${ci_mr_source} (MR ${ci_mr_iid})") - set(ci_profile mr) -elseif(ci_ref_name) - set (CTEST_BUILD_NAME "${ci_ref_name}") - if("${ci_ref_name}" STREQUAL "develop") - set(ci_profile develop) - elseif("${ci_ref_name}" MATCHES "^release-[0-9]+\\.[0-9]+\$") - set(ci_profile release) - endif() -endif() +set_dash_build_name() # set pipelines to enable documentation set(ci_cookbook_profiles mr develop release) @@ -109,7 +95,7 @@ find_program(CTEST_GIT_COMMAND NAMES git git.cmd) # End of configuration -ctest_start (Experimental TRACK Experimental) +ctest_start (Experimental TRACK CI_Build) ctest_update() diff --git a/CI/main_superbuild.cmake b/CI/main_superbuild.cmake index 93c896662166992ebdf1bd0e8cce860f26c2d959..757dbb1ad39e55f75e76f157da0ddd15118c74d7 100644 --- a/CI/main_superbuild.cmake +++ b/CI/main_superbuild.cmake @@ -18,6 +18,8 @@ # limitations under the License. # +include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" ) + # This script is a prototype for the future CI, it may evolve rapidly in a near future set (ENV{LANG} "C") # Only ascii output get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY) @@ -44,7 +46,12 @@ file(READ "${OTB_SOURCE_DIR}/sb_branch.txt" BRANCH_NAME) ############################################################################### set ( REMOTE "https://gitlab.orfeo-toolbox.org/gbonnefille/superbuild-artifact.git") # set ( BRANCH_NAME "${IMAGE_NAME}/${SB_MD5}") -set( GIT "git" ) + +# Look for a GIT command-line client. +find_program(CTEST_GIT_COMMAND NAMES git git.cmd) + +# FIXME: Replace ${GIT} variable with $[CTEST_GIT_COMMAND}" +set( GIT "${CTEST_GIT_COMMAND}" ) execute_process( COMMAND ${GIT} "clone" "${REMOTE}" "--branch" "${BRANCH_NAME}" @@ -69,7 +76,7 @@ endif() set (CMAKE_COMMAND "cmake") execute_process( - COMMAND ${CMAKE_COMMAND} "-E" "tar" "xf" + COMMAND ${CMAKE_COMMAND} "-E" "tar" "xf" "${CI_PROJ_DIR}/superbuild-artifact/SuperBuild_Install.tar" WORKING_DIRECTORY ${CI_ROOT_DIR} ) @@ -92,10 +99,12 @@ endif() set ( CTEST_BUILD_CONFIGURATION "Release" ) set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" ) -set ( CTEST_BUILD_FLAGS "-j16" ) -set ( CTEST_BUILD_NAME "Superbuild_Build_Otb" ) +set ( CTEST_BUILD_FLAGS "-j8" ) set ( CTEST_SITE "${IMAGE_NAME}" ) +# Find the build name and CI profile +set_dash_build_name() + # Directory variable set ( CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}" ) set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" ) @@ -105,25 +114,30 @@ set ( PROJECT_SOURCE_DIR "${OTB_SOURCE_DIR}" ) set (CONFIGURE_OPTIONS "") include ( "${CMAKE_CURRENT_LIST_DIR}/configure_option.cmake" ) # SuperBuild case : one more configure option -set ( CONFIGURE_OPTIONS +set ( CONFIGURE_OPTIONS "${CONFIGURE_OPTIONS}-DCMAKE_PREFIX_PATH=${XDK_PATH};") # Hack because there is no more superbuild available (LIBKML) -set ( CONFIGURE_OPTIONS +set ( CONFIGURE_OPTIONS "${CONFIGURE_OPTIONS}-DOTB_USE_LIBKML:BOOL=OFF;" ) # FIX ME this part might platform dependent set( GDAL_DATA "${XDK_PATH}/share/gdal" ) set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" ) set( PROJ_LIB "${XDK_PATH}/share" ) -set( CTEST_ENVIRONMENT +set( CTEST_ENVIRONMENT "PATH=${XDK_PATH}/lib:${XDK_PATH}/bin:$ENV{PATH} GDAL_DATA= GDAL_DATA GEOTIFF_CSV= GEOTIFF_CSV PROJ_LIB= PROJ_LIB ") -ctest_start (Experimental TRACK Experimental) +# Sources are already checked out : do nothing for update +set(CTEST_GIT_UPDATE_CUSTOM echo No update) + +ctest_start (Experimental TRACK CI_Build) + +ctest_update() ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" SOURCE "${OTB_SOURCE_DIR}" @@ -134,7 +148,7 @@ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" if ( NOT _configure_rv EQUAL 0 ) ctest_submit() - message( SEND_ERROR "An error occurs during ctest_configure.") + message( FATAL_ERROR "An error occurs during ctest_configure.") endif() ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" @@ -143,19 +157,17 @@ ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" ) if ( NOT _build_rv EQUAL 0 ) - ctest_submit() message( SEND_ERROR "An error occurs during ctest_build.") endif() # Uncomment when ready for test -# ctest_test(PARALLEL_LEVEL 8 -# RETURN_VALUE _test_rv -# CAPTURE_CMAKE_ERROR _test_error -# ) - -# if ( NOT _test_rv EQUAL 0 ) -# ctest_submit() -# message( SEND_ERROR "An error occurs during ctest_test.") -# endif() +ctest_test(PARALLEL_LEVEL 8 + RETURN_VALUE _test_rv + CAPTURE_CMAKE_ERROR _test_error + ) + +if ( NOT _test_rv EQUAL 0 ) + message( WARNING "Some tests have failed.") +endif() ctest_submit() diff --git a/CI/prepare_superbuild.cmake b/CI/prepare_superbuild.cmake index c94329788b40a995e5c1a497823fbfc9c8b32743..65f420c5e865e43043e544764c599a494457f242 100644 --- a/CI/prepare_superbuild.cmake +++ b/CI/prepare_superbuild.cmake @@ -19,6 +19,8 @@ # # This script is for the superbuild build on the CI platform +include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" ) + set (ENV{LANG} "C") # Only ascii output get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY) get_filename_component(CI_PROJ_DIR ${OTB_SOURCE_DIR} DIRECTORY) @@ -39,7 +41,10 @@ set ( PROJECT_SOURCE_DIR "${SUPERBUILD_SOURCE_DIR}" ) set ( CTEST_SOURCE_DIRECTORY "${SUPERBUILD_SOURCE_DIR}" ) set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" ) set ( CTEST_SITE "${IMAGE_NAME}" ) -set ( CTEST_BUILD_NAME "Superbuild_Build_Depends" ) # FIXME + + +# Find the build name and CI profile +set_dash_build_name() # We need a directory independent from user # in CI the architecture is /builds/user/otb @@ -50,7 +55,7 @@ set ( CTEST_BUILD_NAME "Superbuild_Build_Depends" ) # FIXME set (CTEST_INSTALL_DIRECTORY "${CI_ROOT_DIR}/xdk/") # HACK -# This is needed because when using return() function ctest is trying +# This is needed because when using return() function ctest is trying # to run the CTEST_COMMAND. And we need it to not produce an error set (CTEST_COMMAND "echo \"Exit\"") # HACK FIX ME set (CMAKE_COMMAND "cmake") @@ -61,12 +66,23 @@ set (CMAKE_COMMAND "cmake") ######################################################################## ######################################################################## -ctest_start (Experimental TRACK Experimental) +# Look for a GIT command-line client. +find_program(CTEST_GIT_COMMAND NAMES git git.cmd) + +# FIXME: Replace ${GIT} variable with $[CTEST_GIT_COMMAND}" +set( GIT "${CTEST_GIT_COMMAND}" ) + +# Sources are already checked out : do nothing for update +set(CTEST_GIT_UPDATE_CUSTOM echo No update) + +ctest_start (Experimental TRACK CI_Prepare) + +ctest_update( SOURCE "${OTB_SOURCE_DIR}" ) set(CTEST_BUILD_FLAGS "-j16") set ( SB_CONFIGURE_OPTIONS "") -include( "${CMAKE_CURRENT_LIST_DIR}/sb_configure_options.cmake" ) +include( "${CMAKE_CURRENT_LIST_DIR}/../SuperBuild/CI/configure_options.cmake" ) ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" SOURCE "${SUPERBUILD_SOURCE_DIR}" @@ -117,9 +133,9 @@ file ( WRITE "${OTB_SOURCE_DIR}/sb_branch.txt" "${IMAGE_NAME}/${SB_MD5}") message( "Checking out git for existence of archive") set ( REMOTE "https://gitlab.orfeo-toolbox.org/gbonnefille/superbuild-artifact/") set ( BRANCH_NAME "${IMAGE_NAME}/${SB_MD5}") -set( GIT "git" ) + execute_process( - COMMAND ${GIT} "ls-remote" "${REMOTE}" "${BRANCH_NAME}" + COMMAND ${GIT} "ls-remote" "${REMOTE}" "${BRANCH_NAME}" OUTPUT_VARIABLE IS_SB_BUILD ) if ( IS_SB_BUILD ) @@ -162,17 +178,17 @@ ctest_submit() # The image used will be passed to this script. # TODO verify that images does not have forbidden char in there name # TODO right now we rely on ctest_build to know whether there has been an error -# in build, whereas SuperBuild does not necessarily return an error if something +# in build, whereas SuperBuild does not necessarily return an error if something #Â goes wrong set ( SB_ARTIFACT_GIT "${CI_PROJ_DIR}/superbuild-artifact" ) -# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are +# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are # different because one is https and one is ssh. Both should be ssh. set( REPOSITORY_GIT_URL "git@gitlab.orfeo-toolbox.org:gbonnefille/superbuild-artifact.git") # We clone master to have a basic configuration, mainly a correct .gitattribute # git clone $REMOTE --branch master --depth 1 superbuild-artifact execute_process( - COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}" + COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}" "--branch" "master" "--depth" "1" "superbuild-artifact" WORKING_DIRECTORY "${CI_PROJ_DIR}" ) @@ -181,7 +197,7 @@ execute_process( # StrictHostKeyChecking so we don't have to add the host as a known key # -F /dev/null so the agent is not taking a default file ~/.ssh/.. execute_process( - COMMAND ${GIT} "config" "core.sshCommand" + COMMAND ${GIT} "config" "core.sshCommand" "ssh -o StrictHostKeyChecking=no -F /dev/null" WORKING_DIRECTORY ${SB_ARTIFACT_GIT} RESULT_VARIABLE ssh_res @@ -248,13 +264,13 @@ set ( SB_TAR_NAME "SuperBuild_Install.tar" ) # We need to create tar in its directory to avoid weird name in file # "tar: Removing leading `../../' from member names" # WARNING -# We are creating a tar containing xdk/.., so when extracting the archive in +# We are creating a tar containing xdk/.., so when extracting the archive in # an other environment the output file will be xdk... Obvious isn't it? # Well... Not for everyone... -# May be for easier maintainability the tar name should be the same as the +# May be for easier maintainability the tar name should be the same as the # file inside. execute_process( - COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "${SB_TAR_NAME}" + COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "${SB_TAR_NAME}" -- "${CTEST_INSTALL_DIRECTORY}" WORKING_DIRECTORY ${CI_ROOT_DIR} ) @@ -295,7 +311,7 @@ endif() # In our case if toto is deploying a key in superbuild-artifact repo # the the mail will be toto's execute_process( - COMMAND ${GIT} "commit" "--author=\"otbbot <otbbot@orfeo-toolbox.org>\"" + COMMAND ${GIT} "commit" "--author=\"otbbot <otbbot@orfeo-toolbox.org>\"" "-m" "\"New Superbuild for ${SB_MD5} on ${IMAGE_NAME}\"" WORKING_DIRECTORY ${SB_ARTIFACT_GIT} RESULT_VARIABLE com_res @@ -342,4 +358,4 @@ if ( DEBUG ) message( "push_res = ${push_res}" ) message( "push_out = ${push_out}" ) message( "push_err = ${push_err}" ) -endif() \ No newline at end of file +endif() diff --git a/CI/sb_configure_options.cmake b/SuperBuild/CI/configure_options.cmake similarity index 95% rename from CI/sb_configure_options.cmake rename to SuperBuild/CI/configure_options.cmake index 1487eedcaf79fce02a113d75d030a6635595c2dc..7b1b1e2e9deb0e7d7780b43edd31425f8c1be895 100644 --- a/CI/sb_configure_options.cmake +++ b/SuperBuild/CI/configure_options.cmake @@ -24,7 +24,8 @@ set ( cmake_configure_option CMAKE_INSTALL_PREFIX:PATH=${CTEST_INSTALL_DIRECTORY}") set ( temporary_option -"OTB_USE_LIBKML=OFF") +"OTB_USE_LIBKML=OFF +OTB_USE_MPI=OFF") set(concat_options "${cmake_configure_option} @@ -36,4 +37,4 @@ string (REPLACE "\n" ";" sb_options ${concat_options}) foreach(item ${sb_options}) set( SB_CONFIGURE_OPTIONS "${SB_CONFIGURE_OPTIONS}-D${item};") -endforeach(item) \ No newline at end of file +endforeach(item) diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index a4d361d3aecbe6c35fd496837d27b1db35070653..27f42a751b0c8966d6775369645e9c39a43b6b80 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -18,6 +18,8 @@ # limitations under the License. # +## COMMMENT to dirty superbuild artifact. TAG: 1. + cmake_minimum_required(VERSION 3.3.0) project(OTB-SuperBuild)