From 2abd6ae7d7d1083e455e42cde116ec1e6ba575a6 Mon Sep 17 00:00:00 2001 From: Nicolas Denoyelle Date: Thu, 2 May 2019 13:23:39 -0500 Subject: [PATCH] Release pipeline ready to be merged. Release works as follow: * events triggering a pipeline on master will create aml-latest dist release * events on branches v.0.0.x will also trigger the pipeline and create a release archive aml-0.0.x.tar.gz * new tags will trigger the release pipeline and if tag match vx.x.x then it will create a release aml-x.x.x.tar.gz * Whenever a branch, tag is updated and a release exists, only the link to dist archives is update. Description will remain the same, but can be updated withe the script release.sh In order to work, the release pipeline still needs to export a RELEASE_TOKEN with rights on gitlab api. --- .gitlab-ci.yml | 35 ++++++- release.sh | 271 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+), 1 deletion(-) create mode 100755 release.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82d34af..c82258d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ variables: stages: - style - build + - dist - release make:generic: @@ -55,11 +56,43 @@ checkpatch: script: - nix run -f "$ARGOPKGS" checkpatch --command checkpatch.pl +dist-artifacts: + stage: dist + when: on_success + only: + - tags + script: + - nix-build "$ARGOPKGS" -A aml-dist --arg aml-src ./. + - echo "$CI_JOB_ID" > .cijobid + artifacts: + when: on_success + paths: + - result/*.tar.gz + - .cijobid + expire_in: 1h + readthedocs: stage: release when: on_success only: - master - - /v\.[0-9]+\.[0-9]+\.[0-9]+/ + - /v\.[0-9]+\.[0-9]+\.x/ script: - nix run nixpkgs.curl -c curl -X POST -d "branch=$CI_COMMIT_REF_NAME" -d "token=$READTHEDOCS_TOKEN" https://readthedocs.org/api/v2/webhook/argo-aml/83161/ + +dist-release: + stage: release + when: on_success + only: + - tags + dependencies: + - dist-artifacts + script: + - CI_ARTIFACT_JOB_ID="$(cat .cijobid)" + - ./release.sh CREATE $CI_ARTIFACT_JOB_ID $CI_PROJECT_ID $RELEASE_TOKEN $CI_COMMIT_REF_NAME + artifacts: + when: on_success + paths: + - result/*.tar.gz + - CHECKSUM + expire_in: 1000y diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..5b0df57 --- /dev/null +++ b/release.sh @@ -0,0 +1,271 @@ +#!/bin/bash + +################################################################################ +# Copyright 2019 UChicago Argonne, LLC. +# (c.f. AUTHORS, LICENSE) +# +# This file is part of the AML project. +# For more info, see https://xgitlab.cels.anl.gov/argo/aml +# +# SPDX-License-Identifier: BSD-3-Clause +################################################################################ + +################################## +# usage: +################################## + +function usage(){ + echo "./release.sh job_id project_id job_token tag (description)" + echo "" + echo "ACTION: Can be one of CREATE, UPDATE or DELETE. If CREATE and" + echo " release exists, then it is updated. If update and release" + echo " does not exists, then it is created." + echo "job_id: the identifier of CI job $CI_JOB_ID which created " + echo " artifacts. Can be found in artifact URL" + echo "project_id: The AML project gitlab id $CI_PROJECT_ID. By this time" + echo " it is 1070." + echo "job_token: The CI job private token $CI_JOB_TOKEN to access" + echo " release API." + echo "tag: The branch or tag of commit $CI_COMMIT_REF_NAME." + echo " It is assumed to be whether "master" or vX.X.X," + echo " otherwise the script will exit." +} + +################################## +# Process Arguments and Set Globals +################################## + +ARGC=5 + +if [ $# -lt $ARGC ]; then + usage + echo "Wrong number($#) of arguments provided. $ARGC arguments required" 1>&2 + echo "Command line was:" + echo "$*" + exit 1 +fi + +JOB_ID=$2 +PROJECT_ID=$3 +JOB_TOKEN=$4 +TAG=$5 + +RELEASE_VERSION="" # set to "latest" or "X.X.X" from $TAG +BASE_URL="https://xgitlab.cels.anl.gov/api/v4/projects" +URL="$BASE_URL/$PROJECT_ID" +TOKEN_HEADER="PRIVATE-TOKEN: $JOB_TOKEN" + +# Check release version match format +if [ "$TAG" = "master" ]; then + RELEASE_VERSION="latest" +else + echo $TAG | grep -q -E 'v[0-9]+\.[0-9]+\.[0-9]+' + if [ $? -ne 0 ]; then + usage + echo "Argument \"$TAG\" is not \"master\" or in format \"vX.X.X\"" 1>&2 + exit 1 + fi + RELEASE_VERSION=$(echo $TAG | cut -c 2-) +fi + +NAME="AML Release $RELEASE_VERSION" + +################################## +# Check Remote Access to Release +################################## + +OUT=$(curl -s --header "PRIVATE-TOKEN: $JOB_TOKEN" "$URL") +if [ -z "$OUT" ]; then + echo "URL:$URL did not return response. Check connection." 1>&2 + exit 1 +fi +if [ ! -z "$(echo $OUT | grep '404 Project Not Found')" ]; then + echo "Project id:$PROJECT_ID is not valid. Please check." 1>&2 + curl --header "PRIVATE-TOKEN: $JOB_TOKEN" "$BASE_URL" + exit 1 +fi +if [ ! -z "$(echo $OUT | grep '401 Unauthorized')" ]; then + echo "Token:$JOB_TOKEN is not valid. Please check." 1>&2 + exit 1 +fi + +# Now will work on releases directory only +URL="$URL/releases" + +################################## +# Look for Artifacts Names and Format +################################## + +EXTENSIONS=("tar.gz" "zip" "bz2" "bz" "tar") +ARTIFACTS_BASE_URL="https://xgitlab.cels.anl.gov/argo/aml/-/jobs/$JOB_ID/artifacts/file" + +#Takes extension name as first argument +function exist_result_ext(){ + if [ ! -d result ]; then + echo "Expected to find './result' directory containing the result of make dist." 1>&2 + return + fi + + for f in $(ls result); do + regex=$(echo .+.$1$) + if [ ! -z $(echo $f | grep -E $regex) ]; then + echo $f + return + fi + done +} + +#Takes extension name as first argument +function artifact_name_from_ext(){ + echo "aml-$RELEASE_VERSION.$1" +} + +function artifact_checksum(){ + sha256sum "result/$1" | cut -d " " -f 1 > CHECKSUM + echo "$ARTIFACTS_BASE_URL/CHECKSUM" +} + +#Takes extension name as first argument +function artifact_url(){ + echo "$ARTIFACTS_BASE_URL/result/$1" +} + +function artifact_upload_from_ext(){ + ART_FILENAME=$(exist_result_ext $1) + ART_NAME=$(artifact_name_from_ext $1) + ART_URL=$(artifact_url $ART_FILENAME) + if [ ! -z "$ART_FILENAME" ]; then + curl --header "$TOKEN_HEADER" \ + --data name="$ART_NAME" \ + --data url="$ART_URL" \ + --request POST "$URL/$TAG/assets/links" + + CHECKSUM_URL=$(artifact_checksum $ART_FILENAME) + + curl --header "$TOKEN_HEADER" \ + --data name="CHECKSUM" \ + --data url="$CHECKSUM_URL" \ + --request POST "$URL/$TAG/assets/links" + fi +} + +function artifacts_get_ids(){ + LINKS=$(curl --header "$TOKEN_HEADER" --request GET "$URL/$TAG/assets/links") + echo $LINKS | grep -oE "\"id\"\:[0-9]+" | cut -c 6- +} + +function artifacts_rm_link(){ + curl --header "$TOKEN_HEADER" \ + --request DELETE "$URL/$TAG/assets/links/$1" +} + +function description_update(){ + curl -s \ + --header "$CONTENT_HEADER" \ + --header "$TOKEN_HEADER" \ + --data name=$NAME \ + --data tag_name=$TAG \ + --data description="$1" \ + --request PUT $URL/$TAG +} + +################################## +# Build curl Commands/Arguments +################################## + +set -x + +# Check if releases exists +curl -s --header "PRIVATE-TOKEN: $JOB_TOKEN" $URL/$TAG | grep -q $RELEASE_VERSION +EXISTS_RELEASE=$? + +function create_release(){ + if [ $EXISTS_RELEASE -eq 0 ]; then + update_release $* + else + CONTENT_HEADER="Content-Type: application/json" + if [ ! -z "$6" ]; then + DESCRIPTION="$6" + else + DESCRIPTION="This AML $TAG Release" + fi + COUNT=0 + ARTIFACTS="" + + for ext in ${EXTENSIONS[@]}; do + ART_FILENAME=$(exist_result_ext $1) + ART_NAME=$(artifact_name_from_ext $1) + ART_URL=$(artifact_url $ART_FILENAME) + if [ ! -z "$ART_FILENAME" ]; then + CHECKSUM_URL=$(artifact_checksum $ART_FILENAME) + ARTIFACTS="$ARTIFACTS { \"name\": \"$ART_NAME\", \"url\": \"$ART_URL\" }, { \"name\": \"CHECKSUM\", \"url\": \"$CHECKSUM_URL\" }" + COUNT=$(($COUNT + 1)) + fi + done + + if [ $COUNT -gt 0 ]; then + ARTIFACTS="{ \"links\": [ ${ARTIFACTS::-2} ] }" + fi + + DATA="{ \"name\": \"$NAME\", \"tag_name\": \"$TAG\", \"ref\": \"$TAG\", \"assets\": $ARTIFACTS, \"description\": \"$DESCRIPTION\"}" + curl -s \ + --header "$CONTENT_HEADER" \ + --header "$TOKEN_HEADER" \ + --data "$DATA" \ + --request POST $URL + fi +} + +# Does not delete existing artifacts but +# replaces pointer to the good artifact. +function update_release(){ + if [ $EXISTS_RELEASE -ne 0 ]; then + create_release $* + else + #Update description + if [ ! -z "$6" ]; then + description_update $6 + fi + + #Delete existing assets + for ID in $(artifacts_get_ids); do + artifacts_rm_link $ID + done + + #Upload new assets + for ext in ${EXTENSIONS[@]}; do + artifact_upload_from_ext $ext + done + fi +} + +# Does not delete existing artifacts +function delete_release(){ + if [ $EXISTS_RELEASE -ne 0 ]; then + echo "Release $TAG cannot be found at $URL." + else + curl -s \ + --header "$TOKEN_HEADER" \ + --request DELETE $URL/$TAG + fi +} + +################################## +# Run Distant Modifications +################################## + +function main(){ + if [ "$1" = "DELETE" ]; then + delete_release $* + elif [ "$1" = "CREATE" ]; then + create_release $* + elif [ "$1" = "UPDATE" ]; then + update_release $* + else + usage + echo "Wrong ACTION argument." 1>&2 + fi +} + +main $* + -- 2.26.2