Commit 65bfe823 authored by Antoine Regimbeau's avatar Antoine Regimbeau
Browse files

Merge branch 'ci_hidden_key' into 'develop'

CI: change git and ssh setup

See merge request orfeotoolbox/otb!489
No related merge requests found
Showing with 22 additions and 87 deletions
+22 -87
...@@ -18,17 +18,6 @@ variables: ...@@ -18,17 +18,6 @@ variables:
# The fetch strategy fails with LFS and GitLab # The fetch strategy fails with LFS and GitLab
GIT_STRATEGY: "clone" GIT_STRATEGY: "clone"
before_script:
# make sure LFS hooks are installed
- git lfs install
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
- git checkout -f -q $CI_COMMIT_SHA
after_script:
- python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_API_TOKEN
stages: stages:
- precheck - precheck
- prepare - prepare
...@@ -47,9 +36,10 @@ fast-build: ...@@ -47,9 +36,10 @@ fast-build:
stage: precheck stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script: before_script:
- export GIT_LFS_SKIP_SMUDGE=1 - git checkout -f -q $CI_COMMIT_SHA
- git checkout $CI_COMMIT_REF_NAME
- python3 CI/check_twin_pipelines.py - python3 CI/check_twin_pipelines.py
after_script:
- python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH $CI_MERGE_REQUEST_REF_PATH
script: script:
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s - ccache -s
...@@ -58,7 +48,14 @@ fast-build: ...@@ -58,7 +48,14 @@ fast-build:
extends: .general extends: .general
only: [merge_requests] only: [merge_requests]
stage: build stage: build
dependencies: [] before_script:
- git lfs install --skip-repo
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
- git checkout -f -q $CI_COMMIT_SHA
after_script:
- python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH $CI_MERGE_REQUEST_REF_PATH
debian-build: debian-build:
extends: .common-build extends: .common-build
...@@ -82,38 +79,29 @@ ubuntu-llvm: ...@@ -82,38 +79,29 @@ ubuntu-llvm:
only: [merge_requests] only: [merge_requests]
stage: prepare stage: prepare
before_script: before_script:
# This override the previous before_script - git checkout -f -q $CI_COMMIT_SHA
- git checkout $CI_COMMIT_REF_NAME - git lfs install --skip-repo
# We are now doing the git-lfs install
# This is done after the checkout so we avoid downloading Data
# But we need it to upload the archive
- git-lfs install
- git config --global user.email "otbbot@orfeo-toolbox.org" - git config --global user.email "otbbot@orfeo-toolbox.org"
- git config --global user.name "otbbot" - git config --global user.name "otbbot"
- eval $(ssh-agent -s)
- ssh-add <(echo "$K8S_SECRET_SSH")
# This is for debug, we are checking the owner of the ssh key
- ssh -o StrictHostKeyChecking=no -T git@gitlab.orfeo-toolbox.org
after_script:
- echo "Nothing to do for after_script"
artifacts: artifacts:
expire_in: 24 hrs expire_in: 24 hrs
when: always
paths: paths:
- sb_branch.txt
# This recovers logs from superbuild build # This recovers logs from superbuild build
- build/*/*/*/*.log - build/*/*/*/*.log
- sb_branch.txt
ubuntu-superbuild-prepare: ubuntu-superbuild-prepare:
extends: .common-prepare extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04 image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script: script:
- ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
centos-superbuild-prepare: centos-superbuild-prepare:
extends: .common-prepare extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6 image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script: script:
- ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
ubuntu-superbuild-build: ubuntu-superbuild-build:
extends: .common-build extends: .common-build
......
...@@ -245,14 +245,14 @@ TODO : ...@@ -245,14 +245,14 @@ TODO :
""" """
if __name__ == "__main__": if __name__ == "__main__":
if ( len(sys.argv) < 5 ): if ( len(sys.argv) < 6 ):
print("Usage : "+sys.argv[0]+" commit_sha1 project_id build_directory token") print("Usage : "+sys.argv[0]+" commit_sha1 project_id build_directory token ref_name")
sys.exit() sys.exit()
if trace: if trace:
print (sys.argv) print (sys.argv)
handler = Handler() handler = Handler()
build_dir = os.path.join( sys.argv[3] , "build/") build_dir = os.path.join( sys.argv[3] , "build/")
# build_dir = sys.argv[3] ref_name = sys.argv[5]
if trace: if trace:
print("build_dir is: " + build_dir) print("build_dir is: " + build_dir)
handler.build_dir = build_dir handler.build_dir = build_dir
...@@ -271,7 +271,7 @@ if __name__ == "__main__": ...@@ -271,7 +271,7 @@ if __name__ == "__main__":
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/" gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
gitlab_url += sys.argv[2] + "/statuses/" + sys.argv[1] gitlab_url += sys.argv[2] + "/statuses/" + sys.argv[1]
params = urllib.parse.urlencode({'name':'cdash:' + handler.site , 'state': state ,\ params = urllib.parse.urlencode({'name':'cdash:' + handler.site , 'state': state ,\
'target_url' : cdash_url , 'description' : error }) 'target_url' : cdash_url , 'description' : error , 'ref' : ref_name })
gitlab_request = urllib.request.Request(gitlab_url) gitlab_request = urllib.request.Request(gitlab_url)
gitlab_request.add_header('PRIVATE-TOKEN' , sys.argv[4] ) gitlab_request.add_header('PRIVATE-TOKEN' , sys.argv[4] )
res = urllib.request.urlopen(gitlab_request, data=params.encode('ascii')) res = urllib.request.urlopen(gitlab_request, data=params.encode('ascii'))
......
...@@ -86,6 +86,7 @@ if ( DEBUG ) ...@@ -86,6 +86,7 @@ if ( DEBUG )
message("Xdk folder exists at ${XDK_PATH}") message("Xdk folder exists at ${XDK_PATH}")
else() else()
message("Something went wrong no folder in ${XDK_PATH}") message("Something went wrong no folder in ${XDK_PATH}")
message("The branch was ${BRANCH_NAME}")
endif() endif()
endif() endif()
......
...@@ -24,8 +24,6 @@ include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" ) ...@@ -24,8 +24,6 @@ include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
set (ENV{LANG} "C") # Only ascii output set (ENV{LANG} "C") # Only ascii output
get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY) get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
set ( DEBUG "1" )
set ( SUPERBUILD_SOURCE_DIR "${OTB_SOURCE_DIR}/SuperBuild" ) set ( SUPERBUILD_SOURCE_DIR "${OTB_SOURCE_DIR}/SuperBuild" )
set ( CTEST_BUILD_CONFIGURATION "Release" ) set ( CTEST_BUILD_CONFIGURATION "Release" )
...@@ -185,55 +183,6 @@ execute_process( ...@@ -185,55 +183,6 @@ execute_process(
) )
set ( SB_ARTIFACT_GIT "${OTB_SOURCE_DIR}/superbuild-artifact" ) set ( SB_ARTIFACT_GIT "${OTB_SOURCE_DIR}/superbuild-artifact" )
# setting up the repo
# StrictHostKeyChecking so we don't have to add the host as a known key
# -F /dev/null so the agent is not taking a default file ~/.ssh/..
execute_process(
COMMAND ${GIT} "config" "core.sshCommand"
"ssh -o StrictHostKeyChecking=no"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE ssh_res
OUTPUT_VARIABLE ssh_out
ERROR_VARIABLE ssh_err
)
if ( DEBUG )
message( "Step 1: ssh")
message( "ssh_res = ${ssh_res}" )
message( "ssh_out = ${ssh_out}" )
message( "ssh_err = ${ssh_err}" )
endif()
execute_process(
COMMAND ${GIT} "config" "user.mail" "otbbot@orfeo-toolbox.org"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE mail_res
OUTPUT_VARIABLE mail_out
ERROR_VARIABLE mail_err
)
if ( DEBUG )
message( "Step 2: mail")
message( "mail_res = ${mail_res}" )
message( "mail_out = ${mail_out}" )
message( "mail_err = ${mail_err}" )
endif()
execute_process(
COMMAND ${GIT} "config" "user.name" "otbbot"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE name_res
OUTPUT_VARIABLE name_out
ERROR_VARIABLE name_err
)
if ( DEBUG )
message( "Step 3: name")
message( "name_res = ${name_res}" )
message( "name_out = ${name_out}" )
message( "name_err = ${name_err}" )
endif()
# create a branche # create a branche
execute_process( execute_process(
COMMAND ${GIT} "checkout" "-b" "${BRANCH_NAME}" COMMAND ${GIT} "checkout" "-b" "${BRANCH_NAME}"
...@@ -261,9 +210,6 @@ execute_process( ...@@ -261,9 +210,6 @@ execute_process(
WORKING_DIRECTORY ${OTB_SOURCE_DIR} WORKING_DIRECTORY ${OTB_SOURCE_DIR}
) )
# In a near futur it might be nice to clean up the mess we made...
if ( DEBUG ) if ( DEBUG )
if (EXISTS "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}") if (EXISTS "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
message("Tar file exists in superbuild_artefact at: ${SB_ARTIFACT_GIT}/${SB_TAR_NAME}") message("Tar file exists in superbuild_artefact at: ${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment