Commit 2abd6ae7 authored by Nicolas Denoyelle's avatar Nicolas Denoyelle

Release pipeline ready to be merged. Release works as follow:

* events triggering a pipeline on master will create aml-latest dist release
* events on branches v.0.0.x will also trigger the pipeline and create a release archive aml-0.0.x.tar.gz
* new tags will trigger the release pipeline and if tag match vx.x.x then it will create a release aml-x.x.x.tar.gz
* Whenever a branch, tag is updated and a release exists, only the link to dist archives is update. Description will remain the same, but can be updated withe the script release.sh

In order to work, the release pipeline still needs to export a RELEASE_TOKEN with rights on gitlab api.
parent 88858039
......@@ -4,6 +4,7 @@ variables:
stages:
- style
- build
- dist
- release
make:generic:
......@@ -55,11 +56,43 @@ checkpatch:
script:
- nix run -f "$ARGOPKGS" checkpatch --command checkpatch.pl
dist-artifacts:
stage: dist
when: on_success
only:
- tags
script:
- nix-build "$ARGOPKGS" -A aml-dist --arg aml-src ./.
- echo "$CI_JOB_ID" > .cijobid
artifacts:
when: on_success
paths:
- result/*.tar.gz
- .cijobid
expire_in: 1h
readthedocs:
stage: release
when: on_success
only:
- master
- /v\.[0-9]+\.[0-9]+\.[0-9]+/
- /v\.[0-9]+\.[0-9]+\.x/
script:
- nix run nixpkgs.curl -c curl -X POST -d "branch=$CI_COMMIT_REF_NAME" -d "token=$READTHEDOCS_TOKEN" https://readthedocs.org/api/v2/webhook/argo-aml/83161/
dist-release:
stage: release
when: on_success
only:
- tags
dependencies:
- dist-artifacts
script:
- CI_ARTIFACT_JOB_ID="$(cat .cijobid)"
- ./release.sh CREATE $CI_ARTIFACT_JOB_ID $CI_PROJECT_ID $RELEASE_TOKEN $CI_COMMIT_REF_NAME
artifacts:
when: on_success
paths:
- result/*.tar.gz
- CHECKSUM
expire_in: 1000y
#!/bin/bash
################################################################################
# Copyright 2019 UChicago Argonne, LLC.
# (c.f. AUTHORS, LICENSE)
#
# This file is part of the AML project.
# For more info, see https://xgitlab.cels.anl.gov/argo/aml
#
# SPDX-License-Identifier: BSD-3-Clause
################################################################################
##################################
# usage:
##################################
function usage(){
echo "./release.sh <ACTION> job_id project_id job_token tag (description)"
echo ""
echo "ACTION: Can be one of CREATE, UPDATE or DELETE. If CREATE and"
echo " release exists, then it is updated. If update and release"
echo " does not exists, then it is created."
echo "job_id: the identifier of CI job $CI_JOB_ID which created "
echo " artifacts. Can be found in artifact URL"
echo "project_id: The AML project gitlab id $CI_PROJECT_ID. By this time"
echo " it is 1070."
echo "job_token: The CI job private token $CI_JOB_TOKEN to access"
echo " release API."
echo "tag: The branch or tag of commit $CI_COMMIT_REF_NAME."
echo " It is assumed to be whether "master" or vX.X.X,"
echo " otherwise the script will exit."
}
##################################
# Process Arguments and Set Globals
##################################
ARGC=5
if [ $# -lt $ARGC ]; then
usage
echo "Wrong number($#) of arguments provided. $ARGC arguments required" 1>&2
echo "Command line was:"
echo "$*"
exit 1
fi
JOB_ID=$2
PROJECT_ID=$3
JOB_TOKEN=$4
TAG=$5
RELEASE_VERSION="" # set to "latest" or "X.X.X" from $TAG
BASE_URL="https://xgitlab.cels.anl.gov/api/v4/projects"
URL="$BASE_URL/$PROJECT_ID"
TOKEN_HEADER="PRIVATE-TOKEN: $JOB_TOKEN"
# Check release version match format
if [ "$TAG" = "master" ]; then
RELEASE_VERSION="latest"
else
echo $TAG | grep -q -E 'v[0-9]+\.[0-9]+\.[0-9]+'
if [ $? -ne 0 ]; then
usage
echo "Argument \"$TAG\" is not \"master\" or in format \"vX.X.X\"" 1>&2
exit 1
fi
RELEASE_VERSION=$(echo $TAG | cut -c 2-)
fi
NAME="AML Release $RELEASE_VERSION"
##################################
# Check Remote Access to Release
##################################
OUT=$(curl -s --header "PRIVATE-TOKEN: $JOB_TOKEN" "$URL")
if [ -z "$OUT" ]; then
echo "URL:$URL did not return response. Check connection." 1>&2
exit 1
fi
if [ ! -z "$(echo $OUT | grep '404 Project Not Found')" ]; then
echo "Project id:$PROJECT_ID is not valid. Please check." 1>&2
curl --header "PRIVATE-TOKEN: $JOB_TOKEN" "$BASE_URL"
exit 1
fi
if [ ! -z "$(echo $OUT | grep '401 Unauthorized')" ]; then
echo "Token:$JOB_TOKEN is not valid. Please check." 1>&2
exit 1
fi
# Now will work on releases directory only
URL="$URL/releases"
##################################
# Look for Artifacts Names and Format
##################################
EXTENSIONS=("tar.gz" "zip" "bz2" "bz" "tar")
ARTIFACTS_BASE_URL="https://xgitlab.cels.anl.gov/argo/aml/-/jobs/$JOB_ID/artifacts/file"
#Takes extension name as first argument
function exist_result_ext(){
if [ ! -d result ]; then
echo "Expected to find './result' directory containing the result of make dist." 1>&2
return
fi
for f in $(ls result); do
regex=$(echo .+.$1$)
if [ ! -z $(echo $f | grep -E $regex) ]; then
echo $f
return
fi
done
}
#Takes extension name as first argument
function artifact_name_from_ext(){
echo "aml-$RELEASE_VERSION.$1"
}
function artifact_checksum(){
sha256sum "result/$1" | cut -d " " -f 1 > CHECKSUM
echo "$ARTIFACTS_BASE_URL/CHECKSUM"
}
#Takes extension name as first argument
function artifact_url(){
echo "$ARTIFACTS_BASE_URL/result/$1"
}
function artifact_upload_from_ext(){
ART_FILENAME=$(exist_result_ext $1)
ART_NAME=$(artifact_name_from_ext $1)
ART_URL=$(artifact_url $ART_FILENAME)
if [ ! -z "$ART_FILENAME" ]; then
curl --header "$TOKEN_HEADER" \
--data name="$ART_NAME" \
--data url="$ART_URL" \
--request POST "$URL/$TAG/assets/links"
CHECKSUM_URL=$(artifact_checksum $ART_FILENAME)
curl --header "$TOKEN_HEADER" \
--data name="CHECKSUM" \
--data url="$CHECKSUM_URL" \
--request POST "$URL/$TAG/assets/links"
fi
}
function artifacts_get_ids(){
LINKS=$(curl --header "$TOKEN_HEADER" --request GET "$URL/$TAG/assets/links")
echo $LINKS | grep -oE "\"id\"\:[0-9]+" | cut -c 6-
}
function artifacts_rm_link(){
curl --header "$TOKEN_HEADER" \
--request DELETE "$URL/$TAG/assets/links/$1"
}
function description_update(){
curl -s \
--header "$CONTENT_HEADER" \
--header "$TOKEN_HEADER" \
--data name=$NAME \
--data tag_name=$TAG \
--data description="$1" \
--request PUT $URL/$TAG
}
##################################
# Build curl Commands/Arguments
##################################
set -x
# Check if releases exists
curl -s --header "PRIVATE-TOKEN: $JOB_TOKEN" $URL/$TAG | grep -q $RELEASE_VERSION
EXISTS_RELEASE=$?
function create_release(){
if [ $EXISTS_RELEASE -eq 0 ]; then
update_release $*
else
CONTENT_HEADER="Content-Type: application/json"
if [ ! -z "$6" ]; then
DESCRIPTION="$6"
else
DESCRIPTION="This AML $TAG Release"
fi
COUNT=0
ARTIFACTS=""
for ext in ${EXTENSIONS[@]}; do
ART_FILENAME=$(exist_result_ext $1)
ART_NAME=$(artifact_name_from_ext $1)
ART_URL=$(artifact_url $ART_FILENAME)
if [ ! -z "$ART_FILENAME" ]; then
CHECKSUM_URL=$(artifact_checksum $ART_FILENAME)
ARTIFACTS="$ARTIFACTS { \"name\": \"$ART_NAME\", \"url\": \"$ART_URL\" }, { \"name\": \"CHECKSUM\", \"url\": \"$CHECKSUM_URL\" }"
COUNT=$(($COUNT + 1))
fi
done
if [ $COUNT -gt 0 ]; then
ARTIFACTS="{ \"links\": [ ${ARTIFACTS::-2} ] }"
fi
DATA="{ \"name\": \"$NAME\", \"tag_name\": \"$TAG\", \"ref\": \"$TAG\", \"assets\": $ARTIFACTS, \"description\": \"$DESCRIPTION\"}"
curl -s \
--header "$CONTENT_HEADER" \
--header "$TOKEN_HEADER" \
--data "$DATA" \
--request POST $URL
fi
}
# Does not delete existing artifacts but
# replaces pointer to the good artifact.
function update_release(){
if [ $EXISTS_RELEASE -ne 0 ]; then
create_release $*
else
#Update description
if [ ! -z "$6" ]; then
description_update $6
fi
#Delete existing assets
for ID in $(artifacts_get_ids); do
artifacts_rm_link $ID
done
#Upload new assets
for ext in ${EXTENSIONS[@]}; do
artifact_upload_from_ext $ext
done
fi
}
# Does not delete existing artifacts
function delete_release(){
if [ $EXISTS_RELEASE -ne 0 ]; then
echo "Release $TAG cannot be found at $URL."
else
curl -s \
--header "$TOKEN_HEADER" \
--request DELETE $URL/$TAG
fi
}
##################################
# Run Distant Modifications
##################################
function main(){
if [ "$1" = "DELETE" ]; then
delete_release $*
elif [ "$1" = "CREATE" ]; then
create_release $*
elif [ "$1" = "UPDATE" ]; then
update_release $*
else
usage
echo "Wrong ACTION argument." 1>&2
fi
}
main $*
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment