diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78c9764d184421f9ff821c3ef24e9e7a25e5044b..1f528d1b674f461d06f53b76a5aa3645e5109449 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,6 +29,7 @@ after_script: - python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH stages: + - precheck - prepare - build @@ -39,6 +40,19 @@ stages: - runner_system_failure - stuck_or_timeout_failure +fast-build: + extends: .general + only: [merge_requests, branches] + stage: precheck + image: $CI_REGISTRY/gpasero/otb/otb-install-ubuntu-native + before_script: + - export GIT_LFS_SKIP_SMUDGE=1 + - git checkout $CI_COMMIT_REF_NAME + - python3 CI/check_twin_pipelines.py + script: + - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast + - ccache -s + debian-build: extends: .general only: [merge_requests] @@ -59,15 +73,10 @@ debian-build: - build/Documentation/Cookbook/latex/CookBook-*.pdf - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2 -# This is needed to have only one pipeline in a merge request context ubuntu-llvm: only: [merge_requests] extends: .common-build -ubuntu-llvm-wip: - except: [merge_requests] - extends: .common-build - .common-prepare: # only: [merge_requests] extends: .general diff --git a/CI/check_twin_pipelines.py b/CI/check_twin_pipelines.py new file mode 100644 index 0000000000000000000000000000000000000000..d2a6113924677266de8cf30f8d35debc8ee2c3ea --- /dev/null +++ b/CI/check_twin_pipelines.py @@ -0,0 +1,88 @@ +# +# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES) +# +# This file is part of Orfeo Toolbox +# +# https://www.orfeo-toolbox.org/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import urllib.request +import urllib.parse +import json +import re +import time + +""" +Send a request to Gitlab and return the answer +The request parameter is added after `project/:id/` +WARNING: when data is given, the request will be a POST +Otherwise, it is a GET +""" +def GitlabRequest(request, project=53, data=None, token=''): + gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/" + gitlab_url+= str(project) + '/' + request + params = None + myHeader = {} + if not data is None: + params = urllib.parse.urlencode(data).encode('ascii') + if token: + myHeader = {'PRIVATE-TOKEN':token} + gitlab_request = urllib.request.Request(gitlab_url, data=params, headers=myHeader) + res = urllib.request.urlopen(gitlab_request) + return json.loads(res.read().decode()) + +""" +Check needed environment parameters +""" +def CheckEnvParameters(params): + for p in params: + if not p in os.environ.keys(): + print("Missing environment variable '"+p+"'") + return False + return True + +""" +Check for any duplicated twin pipeline and cancel it +""" +if __name__ == "__main__": + if not CheckEnvParameters(['CI_COMMIT_SHA']): + sys.exit(1) + env = os.environ + sha1 = env['CI_COMMIT_SHA'] + # are we in a merge_request pipeline ? + if 'CI_MERGE_REQUEST_IID' in env.keys(): + if not CheckEnvParameters(['K8S_SECRET_TWIN_PIPELINE','CI_PROJECT_ID','CI_PIPELINE_ID']): + sys.exit(1) + mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_TWIN_PIPELINE']) + wip_regex = re.compile("^[Ww][Ii][Pp]:") + # is it a "WIP" merge request ? + if wip_regex.search(mrInfo["title"]): + # Yes: cancel the current pipeline + print("Cancel current pipeline "+env['CI_PIPELINE_ID']) + GitlabRequest('pipelines/'+env['CI_PIPELINE_ID']+'/cancel', data={}, \ + project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE']) + time.sleep(180) + print("Error: this pipeline should have been canceled") + sys.exit(1) + else: + # No: cancel any previous "normal" pipeline on the same SHA1 + jres = GitlabRequest('pipelines?sha='+sha1, project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE']) + for item in jres: + if item["id"] < int(env['CI_PIPELINE_ID']) and item["status"] == "running": + print("Cancel pipeline "+str(item["id"])) + jres2 = GitlabRequest('pipelines/'+str(item["id"])+'/cancel', data={}, \ + project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE']) + diff --git a/CI/main_ci.cmake b/CI/main_ci.cmake index 26748e63f9983651c4dd8ff7e53ab0c151e36c66..eba3829c5017383a253e86a2d197ec0f1b860f1a 100644 --- a/CI/main_ci.cmake +++ b/CI/main_ci.cmake @@ -54,6 +54,11 @@ if(NOT DEFINED IMAGE_NAME) endif() set (CTEST_SITE "${IMAGE_NAME}") +# Detect "skip testing" +if(DEFINED ENV{CI_SKIP_TESTING}) + set(ci_skip_testing 1) +endif() + # Directory variable set (CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}") if(BUILD_DIR) @@ -116,10 +121,15 @@ if ( NOT _build_rv EQUAL 0 ) message( SEND_ERROR "An error occurs during ctest_build.") endif() -ctest_test(PARALLEL_LEVEL 8 - RETURN_VALUE _test_rv - CAPTURE_CMAKE_ERROR _test_error - ) +if(ci_skip_testing) + message(STATUS "Skip testing") + set(_test_rv 0) +else() + ctest_test(PARALLEL_LEVEL 8 + RETURN_VALUE _test_rv + CAPTURE_CMAKE_ERROR _test_error + ) +endif() if ( NOT _test_rv EQUAL 0 ) message( SEND_ERROR "An error occurs during ctest_test.") diff --git a/CI/ubuntu-18.04-fast.cmake b/CI/ubuntu-18.04-fast.cmake new file mode 100644 index 0000000000000000000000000000000000000000..5ba38ec80c18fcf420f77160b528c6555ee1d84e --- /dev/null +++ b/CI/ubuntu-18.04-fast.cmake @@ -0,0 +1,35 @@ +# +# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES) +# +# This file is part of Orfeo Toolbox +# +# https://www.orfeo-toolbox.org/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Configuration options for ubuntu-18.04-fast + +set(site_option +"opencv_INCLUDE_DIR:PATH=/usr/include +CMAKE_C_COMPILER:STRING=clang +CMAKE_CXX_COMPILER:STRING=clang++ +CMAKE_EXE_LINKER_FLAGS:STRING=-fuse-ld=lld +CMAKE_MODULE_LINKER_FLAGS:STRING=-fuse-ld=lld +CMAKE_SHARED_LINKER_FLAGS:STRING=-fuse-ld=lld +CMAKE_C_COMPILER_LAUNCHER:STRING=ccache +CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache +OTB_USE_SHARK:BOOL=OFF +BUILD_EXAMPLES:BOOL=OFF") + +set(ci_skip_testing ON)