Skip to content
Snippets Groups Projects
Commit cfcc7f4c authored by Eugene Xiang's avatar Eugene Xiang
Browse files

Merge branch 'master-devops3' into 'master' 

Update aws buildspec and build info for new devops

See merge request !344
parents ccde4726 02d04c5c
Branches
Tags
1 merge request!344Master devops3
Pipeline #308626 passed
Stage: build
Stage: coverage
Stage: containerize
Stage: scan
Stage: deploy
Hide whitespace changes
Inline Side-by-side
gcz-provider/provider/gcz-provider-aws/build-aws/build-info.py
+ 12
− 36
View file @ cfcc7f4c
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import boto3
import json
......@@ -19,12 +7,11 @@ import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
# env - CODEBUILD_RESOLVED_SOURCE_VERSION
parser.add_argument("--commit", type=str, help="")
parser.add_argument("--commitId", type=str, help="")
parser.add_argument("--commitMessage", type=str, help="")
parser.add_argument("--commitAuthor", type=str, help="")
parser.add_argument("--commitDate", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
......@@ -32,7 +19,6 @@ parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
......@@ -46,7 +32,10 @@ parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
commitId = args.commit
commitId = args.commitId
commitMessage = args.commitMessage
commitAuthor = args.commitAuthor
commitDate = args.commitDate
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
......@@ -57,29 +46,16 @@ buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
commitArgs = {
"repositoryName": repoName,
"commitId": commitId
}
commitDetail = {
"commit": ""
}
# get the commit detail
try:
codecommit = boto3.client("codecommit")
commitDetail = codecommit.get_commit(**commitArgs)
except Exception as e:
print("Getting commit information from codecommit failed")
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
"commit": commitDetail["commit"]
"commit-id": commitId,
"commit-message": commitMessage,
"commit-author": commitAuthor,
"commit-date":commitDate
}
print(json.dumps(buildInfo, sort_keys=True, indent=4))
......
gcz-provider/provider/gcz-provider-aws/build-aws/buildspec.yaml
+ 8
− 7
View file @ cfcc7f4c
......@@ -29,7 +29,6 @@ phases:
commands:
# fix error noted here: https://github.com/yarnpkg/yarn/issues/7866
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y -qq > /dev/null
- npm cache clean -f
......@@ -59,9 +58,8 @@ phases:
build:
commands:
- export OUTPUT_DIR="dist"
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- export ECR_TAG_BUILDER=`echo build.builder.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_TAG_PROD=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_TAG_BUILDER=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${COMMIT_ID} | cut -c 1-120`
- export ECR_TAG_PROD=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${COMMIT_ID} | cut -c 1-120`
- export ECR_IMAGE_BUILDER=${ECR_REGISTRY}:${ECR_TAG_BUILDER}
- export ECR_IMAGE_PROD=${ECR_REGISTRY}:${ECR_TAG_PROD}
- export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
......@@ -86,8 +84,11 @@ phases:
- echo "Generate build-info.json"
- |
python gcz-provider/provider/gcz-provider-aws/build-aws/build-info.py \
--branch ${CODEBUILD_SOURCE_VERSION} \
--commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--branch ${BRANCH_NAME} \
--commitId ${COMMIT_ID} \
--commitMessage "\"${COMMIT_MESSAGE}\"" \
--commitAuthor "\"${COMMIT_AUTHOR}\"" \
--commitDate "\"${COMMIT_DATE}\"" \
--buildid ${CODEBUILD_BUILD_ID} \
--buildnumber ${CODEBUILD_BUILD_NUMBER} \
--reponame ${REPO_NAME} \
......@@ -98,4 +99,4 @@ artifacts:
files:
- "**/*"
base-directory: "dist"
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
\ No newline at end of file
name: ${BRANCH_NAME}/${PACKAGE_NAME}.zip
\ No newline at end of file
provider/gcz-transformer-aws/build-aws/build-info.py
+ 12
− 38
View file @ cfcc7f4c
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import boto3
import json
......@@ -19,12 +7,11 @@ import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
# env - CODEBUILD_RESOLVED_SOURCE_VERSION
parser.add_argument("--commit", type=str, help="")
parser.add_argument("--commitId", type=str, help="")
parser.add_argument("--commitMessage", type=str, help="")
parser.add_argument("--commitAuthor", type=str, help="")
parser.add_argument("--commitDate", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
......@@ -32,7 +19,6 @@ parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
......@@ -41,12 +27,13 @@ parser.add_argument("--outdir", type=str, help="")
# full ecr image and tag, and any other artifacts
parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
commitId = args.commit
commitId = args.commitId
commitMessage = args.commitMessage
commitAuthor = args.commitAuthor
commitDate = args.commitDate
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
......@@ -57,29 +44,16 @@ buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
commitArgs = {
"repositoryName": repoName,
"commitId": commitId
}
commitDetail = {
"commit": ""
}
# get the commit detail
try:
codecommit = boto3.client("codecommit")
commitDetail = codecommit.get_commit(**commitArgs)
except Exception as e:
print("Getting commit information from codecommit failed")
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
"commit": commitDetail["commit"]
"commit-id": commitId,
"commit-message": commitMessage,
"commit-author": commitAuthor,
"commit-date":commitDate
}
print(json.dumps(buildInfo, sort_keys=True, indent=4))
......
provider/gcz-transformer-aws/build-aws/buildspec.yaml
+ 8
− 12
View file @ cfcc7f4c
......@@ -34,15 +34,12 @@ phases:
commands:
# fix error noted here: https://github.com/yarnpkg/yarn/issues/7866
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y -qq > /dev/null
- apt-get install -y -qq maven > /dev/null
- java -version
- mvn -version
- mkdir -p /root/.m2
- cp ./provider/gcz-transformer-aws/maven/settings.xml /root/.m2/settings.xml
- export AWS_ACCOUNT_ID=`aws sts get-caller-identity | grep Account | cut -d':' -f 2 | cut -d'"' -f 2`
- export AWS_OSDU_DEV_MAVEN_AUTH_TOKEN=`aws codeartifact get-authorization-token --domain $AWS_OSDU_DEV_MAVEN_DOMAIN --domain-owner $AWS_ACCOUNT_ID --query authorizationToken --output text`
- cp ./.mvn/community-maven.settings.xml /root/.m2/settings.xml
# Install git secrets
#TODO temporarily adding hotfix, due to https://github.com/awslabs/git-secrets/issues/220
- export SAY_LOCATION=`ln -s "$(which echo)" say`
......@@ -56,10 +53,8 @@ phases:
- docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
build:
commands:
- export REPO_NAME=${PWD##*/}
- export OUTPUT_DIR="dist"
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${COMMIT_ID} | cut -c 1-120`
- export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
- export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration
......@@ -74,7 +69,7 @@ phases:
# Build Transformer service
- echo "Building primary service assemblies..."
- mvn -ntp -B clean package -pl .,gcz-transformer-core,provider/gcz-transformer-aws -Ddeployment.environment=prod -DskipTests
- mvn sonar:sonar -pl .,provider/gcz-transformer-aws -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME}
- mvn sonar:sonar -pl .,provider/gcz-transformer-aws -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME} -Dsonar.host.url=${SONAR_URL}
- echo "Build and push docker image..."
- docker buildx create --driver docker-container --use
......@@ -83,15 +78,16 @@ phases:
- echo "Generate build-info.json"
- |
python provider/gcz-transformer-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
python provider/gcz-transformer-aws/build-aws/build-info.py --branch ${BRANCH_NAME} --commitId ${COMMIT_ID} \
--commitMessage "\"${COMMIT_MESSAGE}\"" --commitAuthor "\"${COMMIT_AUTHOR}\"" --commitDate "\"${COMMIT_DATE}\"" \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
artifacts:
files:
- "**/*"
base-directory: "dist"
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
name: ${BRANCH_NAME}/${PACKAGE_NAME}.zip
cache:
paths:
- "/root/.m2/**/*"
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment