diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82d34af39ad8a5844bd912dda4ef68635dc57e05..c82258d3537245e31974bc430559799c3d3a2cd8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ variables: stages: - style - build + - dist - release make:generic: @@ -55,11 +56,43 @@ checkpatch: script: - nix run -f "$ARGOPKGS" checkpatch --command checkpatch.pl +dist-artifacts: + stage: dist + when: on_success + only: + - tags + script: + - nix-build "$ARGOPKGS" -A aml-dist --arg aml-src ./. + - echo "$CI_JOB_ID" > .cijobid + artifacts: + when: on_success + paths: + - result/*.tar.gz + - .cijobid + expire_in: 1h + readthedocs: stage: release when: on_success only: - master - - /v\.[0-9]+\.[0-9]+\.[0-9]+/ + - /v\.[0-9]+\.[0-9]+\.x/ script: - nix run nixpkgs.curl -c curl -X POST -d "branch=$CI_COMMIT_REF_NAME" -d "token=$READTHEDOCS_TOKEN" https://readthedocs.org/api/v2/webhook/argo-aml/83161/ + +dist-release: + stage: release + when: on_success + only: + - tags + dependencies: + - dist-artifacts + script: + - CI_ARTIFACT_JOB_ID="$(cat .cijobid)" + - ./release.sh CREATE $CI_ARTIFACT_JOB_ID $CI_PROJECT_ID $RELEASE_TOKEN $CI_COMMIT_REF_NAME + artifacts: + when: on_success + paths: + - result/*.tar.gz + - CHECKSUM + expire_in: 1000y diff --git a/release.sh b/release.sh new file mode 100755 index 0000000000000000000000000000000000000000..5b0df5796ed1668f1f32427e7b77615a3a6673cd --- /dev/null +++ b/release.sh @@ -0,0 +1,271 @@ +#!/bin/bash + +################################################################################ +# Copyright 2019 UChicago Argonne, LLC. +# (c.f. AUTHORS, LICENSE) +# +# This file is part of the AML project. +# For more info, see https://xgitlab.cels.anl.gov/argo/aml +# +# SPDX-License-Identifier: BSD-3-Clause +################################################################################ + +################################## +# usage: +################################## + +function usage(){ + echo "./release.sh job_id project_id job_token tag (description)" + echo "" + echo "ACTION: Can be one of CREATE, UPDATE or DELETE. If CREATE and" + echo " release exists, then it is updated. If update and release" + echo " does not exists, then it is created." + echo "job_id: the identifier of CI job $CI_JOB_ID which created " + echo " artifacts. Can be found in artifact URL" + echo "project_id: The AML project gitlab id $CI_PROJECT_ID. By this time" + echo " it is 1070." + echo "job_token: The CI job private token $CI_JOB_TOKEN to access" + echo " release API." + echo "tag: The branch or tag of commit $CI_COMMIT_REF_NAME." + echo " It is assumed to be whether "master" or vX.X.X," + echo " otherwise the script will exit." +} + +################################## +# Process Arguments and Set Globals +################################## + +ARGC=5 + +if [ $# -lt $ARGC ]; then + usage + echo "Wrong number($#) of arguments provided. $ARGC arguments required" 1>&2 + echo "Command line was:" + echo "$*" + exit 1 +fi + +JOB_ID=$2 +PROJECT_ID=$3 +JOB_TOKEN=$4 +TAG=$5 + +RELEASE_VERSION="" # set to "latest" or "X.X.X" from $TAG +BASE_URL="https://xgitlab.cels.anl.gov/api/v4/projects" +URL="$BASE_URL/$PROJECT_ID" +TOKEN_HEADER="PRIVATE-TOKEN: $JOB_TOKEN" + +# Check release version match format +if [ "$TAG" = "master" ]; then + RELEASE_VERSION="latest" +else + echo $TAG | grep -q -E 'v[0-9]+\.[0-9]+\.[0-9]+' + if [ $? -ne 0 ]; then + usage + echo "Argument \"$TAG\" is not \"master\" or in format \"vX.X.X\"" 1>&2 + exit 1 + fi + RELEASE_VERSION=$(echo $TAG | cut -c 2-) +fi + +NAME="AML Release $RELEASE_VERSION" + +################################## +# Check Remote Access to Release +################################## + +OUT=$(curl -s --header "PRIVATE-TOKEN: $JOB_TOKEN" "$URL") +if [ -z "$OUT" ]; then + echo "URL:$URL did not return response. Check connection." 1>&2 + exit 1 +fi +if [ ! -z "$(echo $OUT | grep '404 Project Not Found')" ]; then + echo "Project id:$PROJECT_ID is not valid. Please check." 1>&2 + curl --header "PRIVATE-TOKEN: $JOB_TOKEN" "$BASE_URL" + exit 1 +fi +if [ ! -z "$(echo $OUT | grep '401 Unauthorized')" ]; then + echo "Token:$JOB_TOKEN is not valid. Please check." 1>&2 + exit 1 +fi + +# Now will work on releases directory only +URL="$URL/releases" + +################################## +# Look for Artifacts Names and Format +################################## + +EXTENSIONS=("tar.gz" "zip" "bz2" "bz" "tar") +ARTIFACTS_BASE_URL="https://xgitlab.cels.anl.gov/argo/aml/-/jobs/$JOB_ID/artifacts/file" + +#Takes extension name as first argument +function exist_result_ext(){ + if [ ! -d result ]; then + echo "Expected to find './result' directory containing the result of make dist." 1>&2 + return + fi + + for f in $(ls result); do + regex=$(echo .+.$1$) + if [ ! -z $(echo $f | grep -E $regex) ]; then + echo $f + return + fi + done +} + +#Takes extension name as first argument +function artifact_name_from_ext(){ + echo "aml-$RELEASE_VERSION.$1" +} + +function artifact_checksum(){ + sha256sum "result/$1" | cut -d " " -f 1 > CHECKSUM + echo "$ARTIFACTS_BASE_URL/CHECKSUM" +} + +#Takes extension name as first argument +function artifact_url(){ + echo "$ARTIFACTS_BASE_URL/result/$1" +} + +function artifact_upload_from_ext(){ + ART_FILENAME=$(exist_result_ext $1) + ART_NAME=$(artifact_name_from_ext $1) + ART_URL=$(artifact_url $ART_FILENAME) + if [ ! -z "$ART_FILENAME" ]; then + curl --header "$TOKEN_HEADER" \ + --data name="$ART_NAME" \ + --data url="$ART_URL" \ + --request POST "$URL/$TAG/assets/links" + + CHECKSUM_URL=$(artifact_checksum $ART_FILENAME) + + curl --header "$TOKEN_HEADER" \ + --data name="CHECKSUM" \ + --data url="$CHECKSUM_URL" \ + --request POST "$URL/$TAG/assets/links" + fi +} + +function artifacts_get_ids(){ + LINKS=$(curl --header "$TOKEN_HEADER" --request GET "$URL/$TAG/assets/links") + echo $LINKS | grep -oE "\"id\"\:[0-9]+" | cut -c 6- +} + +function artifacts_rm_link(){ + curl --header "$TOKEN_HEADER" \ + --request DELETE "$URL/$TAG/assets/links/$1" +} + +function description_update(){ + curl -s \ + --header "$CONTENT_HEADER" \ + --header "$TOKEN_HEADER" \ + --data name=$NAME \ + --data tag_name=$TAG \ + --data description="$1" \ + --request PUT $URL/$TAG +} + +################################## +# Build curl Commands/Arguments +################################## + +set -x + +# Check if releases exists +curl -s --header "PRIVATE-TOKEN: $JOB_TOKEN" $URL/$TAG | grep -q $RELEASE_VERSION +EXISTS_RELEASE=$? + +function create_release(){ + if [ $EXISTS_RELEASE -eq 0 ]; then + update_release $* + else + CONTENT_HEADER="Content-Type: application/json" + if [ ! -z "$6" ]; then + DESCRIPTION="$6" + else + DESCRIPTION="This AML $TAG Release" + fi + COUNT=0 + ARTIFACTS="" + + for ext in ${EXTENSIONS[@]}; do + ART_FILENAME=$(exist_result_ext $1) + ART_NAME=$(artifact_name_from_ext $1) + ART_URL=$(artifact_url $ART_FILENAME) + if [ ! -z "$ART_FILENAME" ]; then + CHECKSUM_URL=$(artifact_checksum $ART_FILENAME) + ARTIFACTS="$ARTIFACTS { \"name\": \"$ART_NAME\", \"url\": \"$ART_URL\" }, { \"name\": \"CHECKSUM\", \"url\": \"$CHECKSUM_URL\" }" + COUNT=$(($COUNT + 1)) + fi + done + + if [ $COUNT -gt 0 ]; then + ARTIFACTS="{ \"links\": [ ${ARTIFACTS::-2} ] }" + fi + + DATA="{ \"name\": \"$NAME\", \"tag_name\": \"$TAG\", \"ref\": \"$TAG\", \"assets\": $ARTIFACTS, \"description\": \"$DESCRIPTION\"}" + curl -s \ + --header "$CONTENT_HEADER" \ + --header "$TOKEN_HEADER" \ + --data "$DATA" \ + --request POST $URL + fi +} + +# Does not delete existing artifacts but +# replaces pointer to the good artifact. +function update_release(){ + if [ $EXISTS_RELEASE -ne 0 ]; then + create_release $* + else + #Update description + if [ ! -z "$6" ]; then + description_update $6 + fi + + #Delete existing assets + for ID in $(artifacts_get_ids); do + artifacts_rm_link $ID + done + + #Upload new assets + for ext in ${EXTENSIONS[@]}; do + artifact_upload_from_ext $ext + done + fi +} + +# Does not delete existing artifacts +function delete_release(){ + if [ $EXISTS_RELEASE -ne 0 ]; then + echo "Release $TAG cannot be found at $URL." + else + curl -s \ + --header "$TOKEN_HEADER" \ + --request DELETE $URL/$TAG + fi +} + +################################## +# Run Distant Modifications +################################## + +function main(){ + if [ "$1" = "DELETE" ]; then + delete_release $* + elif [ "$1" = "CREATE" ]; then + create_release $* + elif [ "$1" = "UPDATE" ]; then + update_release $* + else + usage + echo "Wrong ACTION argument." 1>&2 + fi +} + +main $* +