Commit a3a3668c authored by Matt Wise's avatar Matt Wise
Browse files

AWS Implementation of Partition Service

commit 1b1c1971 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Thu Sep 24 2020 12:59:53 GMT-0500 (Central Daylight Time) 

    code review feedback


commit 1f30b0d9 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Thu Sep 24 2020 11:52:45 GMT-0500 (Central Daylight Time) 

    add handler for ssm parameter async put.  make sure response isn't returned until ssm params all exist


commit f0b23b95 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Wed Sep 23 2020 15:54:15 GMT-0500 (Central Daylight Time) 

    create bad jwt for no-access tests


commit a3df6cea 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Wed Sep 23 2020 15:23:07 GMT-0500 (Central Daylight Time) 

    add entitlement support for partition


commit 2ec6d558 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Sep 22 2020 14:32:06 GMT-0500 (Central Daylight Time) 

    update int tests


commit ba6979a6 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Mon Sep 21 2020 21:39:40 GMT-0500 (Central Daylight Time) 

    add springboot actuator health check


commit 541c3b87 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Mon Sep 21 2020 15:23:24 GMT-0500 (Central Daylight Time) 

    fix import for AWSCognitoClient


commit 1c249655 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Mon Sep 21 2020 14:36:16 GMT-0500 (Central Daylight Time) 

    Work on integration tests


commit c45e1435 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Mon Sep 21 2020 12:16:21 GMT-0500 (Central Daylight Time) 

    Add copyright


commit bd71cc33 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Mon Sep 21 2020 12:16:08 GMT-0500 (Central Daylight Time) 

    Add dummy cache implementation


commit 63f035ee 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Mon Sep 21 2020 09:33:23 GMT-0500 (Central Daylight Time) 

    for now, use basic auth. service should be protected by keeping it in a private subnet


commit d575c5b3 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Mon Sep 21 2020 09:16:19 GMT-0500 (Central Daylight Time) 

    surround environment var with braces


commit a9deff7e 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Wed Sep 09 2020 15:55:19 GMT-0500 (Central Daylight Time) 

    add basic properties


commit bf7fbc00 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Wed Sep 09 2020 15:48:19 GMT-0500 (Central Daylight Time) 

    work on delete implementation merging common code


commit 7364962a 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Sep 08 2020 17:36:28 GMT-0500 (Central Daylight Time) 

    work on SSM implementation for partition


commit 02f133a7 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Fri Sep 04 2020 16:38:05 GMT-0500 (Central Daylight Time) 

    boilerplate implementation of AWS Partition Service


commit b740ef22 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Fri Sep 04 2020 15:28:52 GMT-0500 (Central Daylight Time) 

    start partition implementation
parent 3f64f421
......@@ -44,3 +44,7 @@ load-tests/*.pyc
# VSCode
.vscode/
.DS_STORE
dist/
\ No newline at end of file
......@@ -86,6 +86,7 @@
<modules>
<module>partition-core</module>
<module>provider/partition-azure</module>
<module>provider/partition-aws</module>
</modules>
<distributionManagement>
......@@ -128,5 +129,17 @@
<module>provider/partition-azure</module>
</modules>
</profile>
<profile>
<id>partition-aws</id>
<activation>
<property>
<name>env</name>
<value>partition-aws</value>
</property>
</activation>
<modules>
<module>provider/partition-aws</module>
</modules>
</profile>
</profiles>
</project>
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
FROM amazoncorretto:8
ARG JAR_FILE=provider/partition-aws/target/*spring-boot.jar
WORKDIR /
COPY ${JAR_FILE} app.jar
EXPOSE 8080
ENTRYPOINT java $JAVA_OPTS -jar /app.jar
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import boto3
import json
import os
import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
# env - CODEBUILD_RESOLVED_SOURCE_VERSION
parser.add_argument("--commit", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
parser.add_argument("--outdir", type=str, help="")
# full ecr image and tag, and any other artifacts
parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
commitId = args.commit
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
outputDir = args.outdir
artifacts = args.artifact
buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
commitArgs = {
"repositoryName": repoName,
"commitId": commitId
}
commitDetail = {
"commit": ""
}
# get the commit detail
try:
codecommit = boto3.client("codecommit")
commitDetail = codecommit.get_commit(**commitArgs)
except Exception as e:
print("Getting commit information from codecommit failed")
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
"commit": commitDetail["commit"]
}
print(json.dumps(buildInfo, sort_keys=True, indent=4))
# write the build.json file to dist
f = open(buildInfoFilePath, "w")
f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
f.close()
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
version: 0.2
phases:
install:
runtime-versions:
java: corretto8
commands:
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
- apt-get install -y maven
- java -version
- mvn -version
- mkdir -p /root/.m2
- cp ./provider/partition-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- export AWS_ACCOUNT_ID=`aws sts get-caller-identity | grep Account | cut -d':' -f 2 | cut -d'"' -f 2`
- export AWS_OSDU_DEV_MAVEN_AUTH_TOKEN=`aws codeartifact get-authorization-token --domain $AWS_OSDU_DEV_MAVEN_DOMAIN --domain-owner $AWS_ACCOUNT_ID --query authorizationToken --output text`
pre_build:
commands:
- echo "Logging in to Amazon ECR..."
- $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI
build:
commands:
- export REPO_NAME=${PWD##*/}
- export OUTPUT_DIR="dist"
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
- export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration
- export INTEGRATION_TEST_OUTPUT_BIN=${INTEGRATION_TEST_OUTPUT}/bin
- mkdir -p ${OUTPUT_DIR}/bin
- mkdir -p ${OUTPUT_DIR}/testing && mkdir -p ${INTEGRATION_TEST_OUTPUT} && mkdir -p ${INTEGRATION_TEST_OUTPUT}/bin
- echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
- printenv
- echo "Building primary service assemblies..."
- mvn -B test install -pl partition-core,provider/partition-aws -Ddeployment.environment=prod
- echo "Building integration testing assemblies and gathering artifacts..."
- ./testing/partition-test-aws/build-aws/prepare-dist.sh
- echo "Building docker image..."
- docker build -f provider/partition-aws/build-aws/Dockerfile -t ${ECR_IMAGE} .
- docker tag ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST}
- echo "Pushing docker image..."
- docker push ${ECR_IMAGE}
- docker push ${ECR_IMAGE_BRANCH_LATEST}
- echo "Generate build-info.json"
- |
python provider/partition-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
reports:
SurefireReports: # CodeBuild will create a report group called "SurefireReports".
files: #Store all of the files
- "partition-core/target/surefire-reports/**/*"
- "provider/partition-aws/target/surefire-reports/**/*"
base-directory: "." # Location of the reports
artifacts:
files:
- "**/*"
base-directory: "dist"
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
cache:
paths:
- "/root/.m2/**/*"
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<profiles>
<profile>
<id>aws-osdu-dev-maven</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<repositories>
<repository>
<id>aws-osdu-dev-maven</id>
<url>${env.AWS_OSDU_DEV_MAVEN_URL}</url>
</repository>
<repository>
<id>gitlab-os-core-common-maven</id>
<url>https://community.opengroup.org/api/v4/projects/67/packages/maven</url>
</repository>
</repositories>
</profile>
<profile>
<id>credentialsConfiguration</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<deployment.environment>dev</deployment.environment>
<aws.accessKeyId>no-default</aws.accessKeyId>
<aws.secretKey>no-default</aws.secretKey>
<azure.devops.username>Another-Access-Token-2021</azure.devops.username>
<azure.devops.token>no-default</azure.devops.token>
</properties>
</profile>
</profiles>
<servers>
<server>
<id>aws-osdu-dev-maven</id>
<username>aws</username>
<password>${env.AWS_OSDU_DEV_MAVEN_AUTH_TOKEN}</password>
</server>
</servers>
<!-- CodeArtifact doesn't support external repos yet that aren't Maven Central. ETA Q4 2020. -->
<!-- <mirrors> -->
<!-- <mirror> -->
<!-- <id>aws-osdu-dev-maven</id> -->
<!-- <name>aws-osdu-dev-maven</name> -->
<!-- <url>https://osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url> -->
<!-- <mirrorOf>*,!gitlab-os-core-common-maven</mirrorOf> -->
<!-- </mirror> -->
<!-- </mirrors> -->
<activeProfiles>
<activeProfile>credentialsConfiguration</activeProfile>
</activeProfiles>
</settings>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>partition</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>1.0.0</version>
<relativePath>../../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>partition-aws</artifactId>
<version>1.0.0</version>
<description>Partition service on AWS</description>
<packaging>jar</packaging>
<properties>
<aws.version>1.11.637</aws.version>
<deployment.environment>dev</deployment.environment>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
<version>${aws.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!-- Internal packages -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.7</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>partition-core</artifactId>
<version>1.0.0</version>
</dependency>
<!-- AWS managed packages -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>${aws.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-sns</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-cognitoidentity</artifactId>
</dependency>
<!-- Third party Apache 2.0 license packages -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-jose</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-commons</artifactId>
<version>2.1.10.RELEASE</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
<version>1</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!-- Testing packages -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>2.25.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<version>2.0.2</version>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<version>2.0.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
</exclusion>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.partition.provider.aws.PartitionApplication</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
// Copyright © 2020, Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.partition.provider.aws;
import javax.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class AwsServiceConfig {
@Value("${aws.resource.prefix}")
public String environment;
public String ssmPartitionPrefix;
@PostConstruct
public void init() {
ssmPartitionPrefix = "/osdu/" + environment + "/partition/partitions/";
}
}
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.partition.provider.aws;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@ComponentScan({"org.opengroup.osdu"})
@SpringBootApplication
public class PartitionApplication {
public static void main(String[] args) {
SpringApplication.run(PartitionApplication.class, args);
}
}
\ No newline at end of file
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.partition.provider.aws.cache;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.entitlements.Groups;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.util.Crc32c;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class GroupCache extends RedisCache<String, Groups> {
public GroupCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_GROUP_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_GROUP_PORT) {
super(REDIS_GROUP_HOST, Integer.parseInt(REDIS_GROUP_PORT), 30, String.class, Groups.class);
}
public static String getGroupCacheKey(DpsHeaders headers) {
String key = String.format("entitlement-groups:%s:%s", headers.getPartitionIdWithFallbackToAccountId(),
headers.getAuthorization());
return Crc32c.hashToBase64EncodedString(key);
}
public static String getPartitionGroupsCacheKey(String dataPartitionId) {
String key = String.format("entitlement-groups:data-partition:%s", dataPartitionId);