Commit d54e4b7a authored by Matt Wise's avatar Matt Wise Committed by Sutton
Browse files

Merge Add AWS Implementation

commit 7b90813f 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Fri Aug 21 2020 14:30:03 GMT-0500 (Central Daylight Time) 

    add proper copyright/license


commit 0e325d89 
Author: Kyle Longhurst <kyle.longhurst@parivedasolutions.com> 
Date: Tue Aug 04 2020 14:29:47 GMT-0500 (Central Daylight Time) 

    add aws impl
parent 6d03c58a
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2020 Google LLC
~ Copyright © 2020 Amazon Web Services
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
......@@ -37,6 +38,7 @@
<modules>
<module>workflow-core</module>
<module>provider/workflow-aws</module>
<module>provider/workflow-azure</module>
<module>provider/workflow-gcp</module>
<module>provider/workflow-ibm</module>
......
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
FROM amazoncorretto:8
ARG JAR_FILE=provider/workflow-aws/target/*spring-boot.jar
WORKDIR /
COPY ${JAR_FILE} app.jar
EXPOSE 8080
ENTRYPOINT java $JAVA_OPTS -jar /app.jar
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import boto3
import json
import os
import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
# env - CODEBUILD_RESOLVED_SOURCE_VERSION
parser.add_argument("--commit", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
parser.add_argument("--outdir", type=str, help="")
# full ecr image and tag, and any other artifacts
parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
commitId = args.commit
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
outputDir = args.outdir
artifacts = args.artifact
buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
commitArgs = {
"repositoryName": repoName,
"commitId": commitId
}
commitDetail = {
"commit": ""
}
# get the commit detail
try:
codecommit = boto3.client("codecommit")
commitDetail = codecommit.get_commit(**commitArgs)
except Exception as e:
print("Getting commit information from codecommit failed")
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
"commit": commitDetail["commit"]
}
print(json.dumps(buildInfo, sort_keys=True, indent=4))
# write the build.json file to dist
f = open(buildInfoFilePath, "w")
f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
f.close()
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
version: 0.2
phases:
install:
runtime-versions:
java: corretto8
commands:
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
- apt-get install -y maven
- java -version
- mvn -version
- echo $JAVA_HOME #WHY
- mkdir -p /root/.m2
- cp ./provider/workflow-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- export AWS_OSDU_DEV_MAVEN_AUTH_TOKEN=`aws codeartifact get-authorization-token --domain osdu-dev --domain-owner 888733619319 --query authorizationToken --output text`
pre_build:
commands:
- echo "Logging in to Amazon ECR..."
- $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI
build:
commands:
- export REPO_NAME=${PWD##*/}
- export OUTPUT_DIR="dist"
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
- export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration
- export INTEGRATION_TEST_OUTPUT_BIN=${INTEGRATION_TEST_OUTPUT}/bin
- mkdir -p ${OUTPUT_DIR}/bin
- mkdir -p ${OUTPUT_DIR}/testing && mkdir -p ${INTEGRATION_TEST_OUTPUT} && mkdir -p ${INTEGRATION_TEST_OUTPUT}/bin
- echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
- printenv
- echo "Building primary service assemblies..."
- mvn -B test install -pl workflow-core,provider/workflow-aws -Ddeployment.environment=prod
# Suspended until further notice
# - echo "Copying assemblies to dist..."
# - cp ./provider/workflow-aws/target/*spring-boot.jar ${OUTPUT_DIR}/bin # copy aws jars
# - cp ./workflow-core/target/*.jar ${OUTPUT_DIR}/bin # copy core jar
- echo "Building integration testing assemblies and gathering artifacts..."
- ./testing/workflow-test-aws/build-aws/prepare-dist.sh
- echo "Building docker image..."
- docker build -f provider/workflow-aws/build-aws/Dockerfile -t ${ECR_IMAGE} .
- docker tag ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST}
- echo "Pushing docker image..."
- docker push ${ECR_IMAGE}
- docker push ${ECR_IMAGE_BRANCH_LATEST}
- echo "Generate build-info.json"
- |
python provider/workflow-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
reports:
SurefireReports: # CodeBuild will create a report group called "SurefireReports".
files: #Store all of the files
- "workflow-core/target/surefire-reports/**/*"
- "provider/workflow-aws/target/surefire-reports/**/*"
base-directory: "." # Location of the reports
artifacts:
files:
- "**/*"
base-directory: "dist"
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
cache:
paths:
- "/root/.m2/**/*"
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<profiles>
<profile>
<id>aws-osdu-dev-maven</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<repositories>
<repository>
<id>aws-osdu-dev-maven</id>
<url>${env.AWS_OSDU_DEV_MAVEN_URL}</url>
</repository>
<repository>
<id>gitlab-os-core-common-maven</id>
<url>https://community.opengroup.org/api/v4/projects/67/packages/maven</url>
</repository>
</repositories>
</profile>
<profile>
<id>credentialsConfiguration</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<deployment.environment>dev</deployment.environment>
<aws.accessKeyId>no-default</aws.accessKeyId>
<aws.secretKey>no-default</aws.secretKey>
<azure.devops.username>Another-Access-Token-2021</azure.devops.username>
<azure.devops.token>no-default</azure.devops.token>
</properties>
</profile>
</profiles>
<servers>
<server>
<id>aws-osdu-dev-maven</id>
<username>aws</username>
<password>${env.AWS_OSDU_DEV_MAVEN_AUTH_TOKEN}</password>
</server>
</servers>
<!-- CodeArtifact doesn't support external repos yet that aren't Maven Central. ETA Q4 2020. -->
<!-- <mirrors> -->
<!-- <mirror> -->
<!-- <id>aws-osdu-dev-maven</id> -->
<!-- <name>aws-osdu-dev-maven</name> -->
<!-- <url>https://osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url> -->
<!-- <mirrorOf>*,!gitlab-os-core-common-maven</mirrorOf> -->
<!-- </mirror> -->
<!-- </mirrors> -->
<activeProfiles>
<activeProfile>credentialsConfiguration</activeProfile>
</activeProfiles>
</settings>
<?xml version="1.0" encoding="UTF-8" ?>
<!-- Copyright © 2020 Amazon Web Services-->
<!-- -->
<!-- Licensed under the Apache License, Version 2.0 (the "License");-->
<!-- you may not use this file except in compliance with the License.-->
<!-- You may obtain a copy of the License at-->
<!-- -->
<!-- http://www.apache.org/licenses/LICENSE-2.0-->
<!-- -->
<!-- Unless required by applicable law or agreed to in writing, software-->
<!-- distributed under the License is distributed on an "AS IS" BASIS,-->
<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-->
<!-- See the License for the specific language governing permissions and-->
<!-- limitations under the License.-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>workflow-aws</artifactId>
<version>${version.number}</version>
<packaging>jar</packaging>
<name>workflow-aws</name>
<description>AWS implementation of Workflow service APIs</description>
<properties>
<spring-boot.repackage.skip>true</spring-boot.repackage.skip>
<version.number>0.0.1-SNAPSHOT</version.number>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.12.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.4</version>
</dependency>
<!-- Internal packages -->
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.7</version>
</dependency>
<!-- Testing packages -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.workflow.WorkflowApplication</mainClass>
</configuration>
</plugin>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<configuration>
<keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
<createDependencyReducedPom>true</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<goals>
<goal>shade</goal>
</goals>
<phase>package</phase>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.handlers</resource>
</transformer>
<transformer implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
<resource>META-INF/spring.factories</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.opengroup.osdu.workflow.WorkflowApplication</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##### Sample .env file ###########################################################
#
# Basic use: duplicate this file, and make sure the new copy is also in the root of the AWS
# 'provider' folder, and name it `.env`. Note that on macOS, by default, files starting with
# are considered hidden system files, and are not displayed by default in Finder or the file
# selector (which you will need to use when adding the environment file(s) to the run
# configuration(s). While you can change a setting to show hidden files and folders by
# default, there is also a keyboard shortcut to quickly toggle between hide/show. With either
# Finder as the active application ("Finder" appears next to the Apple logo in the Menu Bar),
# press: command + shift + . (period). You can store configurations for multiple environments
# by adding more duplicates following a naming scheme of your choosing, for example:
# `staging.env`, `uat.env`, or `local.env`.
#
# This requires installing a plugin to your IDE that allows you to use a .env
# file in your repository folder (does NOT get checked into source control;
# only the sample environment configuration (sample.env) should be committed.
#
# Download links for .env file plugins:
# IntelliJ - https://github.com/Ashald/EnvFile
##### Authentication / Secrets #####
# Replace placeholder text with your own AWS secret access keys
# and rename to `.env` - do NOT check-in .env with your credentials! Leave it in .gitignore
AWS_ACCESS_KEY_ID="REPLACE_WITH_YOUR_ACCESS_KEY"
AWS_SECRET_KEY="REPLACE_WITH_YOUR_SECRET_KEY"
AWS_ACCOUNT_ID="REPLACE WITH YOUR ACCOUNT_ID"
#### Urls/Ports #############
AIRFLOW_BASEURL=http://localhost
##### Other environment variables ##########################################################
JAVA_HEAP_MEMORY=4096
ENVIRONMENT=dev
AWS_REGION=us-east-1
ENTITLEMENTS_BASE_URL=http://localhost
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.aws;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "org.opengroup.osdu.workflow")
public class WorkflowAwsApplication {
public static void main(String[] args) {
SpringApplication.run(WorkflowAwsApplication.class, args);
}
}
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.aws.repository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.common.model.WorkflowType;
import org.opengroup.osdu.workflow.aws.util.dynamodb.converters.IngestionStrategyDoc;
import org.opengroup.osdu.workflow.model.IngestionStrategy;
import org.opengroup.osdu.workflow.provider.interfaces.IIngestionStrategyRepository;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import javax.annotation.PostConstruct;
// TODO Will be moved to registry service
@Repository
@Slf4j
@RequiredArgsConstructor
public class IngestionStrategyRepositoryImpl implements IIngestionStrategyRepository {
@Value("${aws.dynamodb.table.prefix}")
String tablePrefix;
@Value("${aws.dynamodb.region}")
String dynamoDbRegion;
@Value("${aws.dynamodb.endpoint}")
String dynamoDbEndpoint;
private DynamoDBQueryHelper queryHelper;
@PostConstruct
public void init() {
queryHelper = new DynamoDBQueryHelper(dynamoDbEndpoint, dynamoDbRegion, tablePrefix);
}
@Override
public IngestionStrategy findByWorkflowTypeAndDataTypeAndUserId(WorkflowType workflowType, String dataType, String userId) {
String compositeKey = String.join(":", workflowType.toString(), dataType, userId);
IngestionStrategyDoc doc = queryHelper.loadByPrimaryKey(IngestionStrategyDoc.class, compositeKey);
if (doc != null) {
IngestionStrategy ingestionStrategy = new IngestionStrategy();
ingestionStrategy.setWorkflowType(WorkflowType.valueOf(doc.getWorkflowType()));
ingestionStrategy.setDataType(dataType);
ingestionStrategy.setUserId(doc.getUserId());
ingestionStrategy.setDagName(doc.getDagName());
return ingestionStrategy;
} else {