Merge branch 'ci_hidden_key' into 'develop'

CI: change git and ssh setup

See merge request orfeotoolbox/otb!489

.gitlab-ci.yml

@@ -18,17 +18,6 @@ variables:
   # The fetch strategy fails with LFS and GitLab
   GIT_STRATEGY: "clone"
 
-before_script:
-  # make sure LFS hooks are installed
-  - git lfs install
-  # Provision efficiently the local LFS cache before checkout
-  - git lfs fetch origin $CI_COMMIT_SHA
-  # Checkout the expected branch
-  - git checkout -f -q $CI_COMMIT_SHA
-
-after_script:
-  - python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_API_TOKEN
-
 stages:
   - precheck
   - prepare
@@ -47,9 +36,10 @@ fast-build:
   stage: precheck
   image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
   before_script:
-    - export GIT_LFS_SKIP_SMUDGE=1
-    - git checkout $CI_COMMIT_REF_NAME
+    - git checkout -f -q $CI_COMMIT_SHA
     - python3 CI/check_twin_pipelines.py
+  after_script:
+    - python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH $CI_MERGE_REQUEST_REF_PATH
   script:
     - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
     - ccache -s
@@ -58,7 +48,14 @@ fast-build:
   extends: .general
   only: [merge_requests]
   stage: build
-  dependencies: []
+  before_script:
+    - git lfs install --skip-repo
+    # Provision efficiently the local LFS cache before checkout
+    - git lfs fetch origin $CI_COMMIT_SHA
+    # Checkout the expected branch
+    - git checkout -f -q $CI_COMMIT_SHA
+  after_script:
+    - python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH $CI_MERGE_REQUEST_REF_PATH
 
 debian-build:
   extends: .common-build
@@ -82,38 +79,29 @@ ubuntu-llvm:
   only: [merge_requests]
   stage: prepare
   before_script:
-# This override the previous before_script
-    - git checkout $CI_COMMIT_REF_NAME
-# We are now doing the git-lfs install
-# This is done after the checkout so we avoid downloading Data
-# But we need it to upload the archive
-    - git-lfs install
+    - git checkout -f -q $CI_COMMIT_SHA
+    - git lfs install --skip-repo
     - git config --global user.email "otbbot@orfeo-toolbox.org"
     - git config --global user.name "otbbot"
-    - eval $(ssh-agent -s)
-    - ssh-add <(echo "$K8S_SECRET_SSH")
-# This is for debug, we are checking the owner of the ssh key
-    - ssh -o StrictHostKeyChecking=no -T git@gitlab.orfeo-toolbox.org
-  after_script:
-    - echo "Nothing to do for after_script"
   artifacts:
     expire_in: 24 hrs
+    when: always
     paths:
+      - sb_branch.txt
       # This recovers logs from superbuild build
       - build/*/*/*/*.log
-      - sb_branch.txt
 
 ubuntu-superbuild-prepare:
   extends: .common-prepare
   image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
   script:
-    - ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
+    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
 
 centos-superbuild-prepare:
   extends: .common-prepare
   image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
   script:
-    - ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
+    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
 
 ubuntu-superbuild-build:
   extends: .common-build

CI/cdash_handler.py

@@ -245,14 +245,14 @@ TODO :
 
 """
 if __name__ == "__main__":
-  if ( len(sys.argv) < 5 ):
-    print("Usage : "+sys.argv[0]+" commit_sha1 project_id build_directory token")
+  if ( len(sys.argv) < 6 ):
+    print("Usage : "+sys.argv[0]+" commit_sha1 project_id build_directory token ref_name")
     sys.exit()
   if trace:
     print (sys.argv)
   handler = Handler()
   build_dir = os.path.join( sys.argv[3] , "build/")
-  # build_dir = sys.argv[3]
+  ref_name = sys.argv[5]
   if trace:
     print("build_dir is: " + build_dir)
   handler.build_dir = build_dir
@@ -271,7 +271,7 @@ if __name__ == "__main__":
   gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
   gitlab_url += sys.argv[2] + "/statuses/" + sys.argv[1]
   params = urllib.parse.urlencode({'name':'cdash:' + handler.site , 'state': state ,\
-   'target_url' : cdash_url , 'description' : error })
+   'target_url' : cdash_url , 'description' : error , 'ref' : ref_name })
   gitlab_request = urllib.request.Request(gitlab_url)
   gitlab_request.add_header('PRIVATE-TOKEN' , sys.argv[4] )
   res = urllib.request.urlopen(gitlab_request, data=params.encode('ascii'))

CI/main_superbuild.cmake

@@ -86,6 +86,7 @@ if ( DEBUG )
     message("Xdk folder exists at ${XDK_PATH}")
   else()
     message("Something went wrong no folder in ${XDK_PATH}")
+    message("The branch was ${BRANCH_NAME}")
   endif()
 endif()
 

CI/prepare_superbuild.cmake

@@ -24,8 +24,6 @@ include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
 set (ENV{LANG} "C") # Only ascii output
 get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
 
-set ( DEBUG "1" )
-
 set ( SUPERBUILD_SOURCE_DIR "${OTB_SOURCE_DIR}/SuperBuild" )
 
 set ( CTEST_BUILD_CONFIGURATION "Release" )
@@ -185,55 +183,6 @@ execute_process(
   )
 set ( SB_ARTIFACT_GIT "${OTB_SOURCE_DIR}/superbuild-artifact" )
 
-# setting up the repo
-# StrictHostKeyChecking so we don't have to add the host as a known key
-# -F /dev/null so the agent is not taking a default file ~/.ssh/..
-execute_process(
-  COMMAND ${GIT} "config" "core.sshCommand"
-  "ssh -o StrictHostKeyChecking=no"
-  WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
-  RESULT_VARIABLE ssh_res
-  OUTPUT_VARIABLE ssh_out
-  ERROR_VARIABLE ssh_err
-  )
-
-if ( DEBUG )
-  message( "Step 1: ssh")
-  message( "ssh_res = ${ssh_res}" )
-  message( "ssh_out = ${ssh_out}" )
-  message( "ssh_err = ${ssh_err}" )
-endif()
-
-execute_process(
-  COMMAND ${GIT} "config" "user.mail" "otbbot@orfeo-toolbox.org"
-  WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
-  RESULT_VARIABLE mail_res
-  OUTPUT_VARIABLE mail_out
-  ERROR_VARIABLE mail_err
-  )
-
-if ( DEBUG )
-  message( "Step 2: mail")
-  message( "mail_res = ${mail_res}" )
-  message( "mail_out = ${mail_out}" )
-  message( "mail_err = ${mail_err}" )
-endif()
-
-execute_process(
-  COMMAND ${GIT} "config" "user.name" "otbbot"
-  WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
-  RESULT_VARIABLE name_res
-  OUTPUT_VARIABLE name_out
-  ERROR_VARIABLE name_err
-  )
-
-if ( DEBUG )
-  message( "Step 3: name")
-  message( "name_res = ${name_res}" )
-  message( "name_out = ${name_out}" )
-  message( "name_err = ${name_err}" )
-endif()
-
 # create a branche
 execute_process(
   COMMAND ${GIT} "checkout" "-b" "${BRANCH_NAME}"
@@ -261,9 +210,6 @@ execute_process(
   WORKING_DIRECTORY ${OTB_SOURCE_DIR}
   )
 
-
-# In a near futur it might be nice to clean up the mess we made...
-
 if ( DEBUG )
   if (EXISTS "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
     message("Tar file exists in superbuild_artefact at: ${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")