Commit 05dc8fad authored by Matt Wise's avatar Matt Wise
Browse files

Merge branch 'feat/aws-impl' into 'master'

Feat/aws impl

See merge request !9
parents 639fa186 b8901d31
Pipeline #12561 passed with stages
in 12 minutes and 32 seconds
......@@ -24,3 +24,6 @@ analyze:
type: mvn
target: provider/crs-catalog-gcp/crs-catalog-gke/pom.xml
path: .
- name: crs-catalog-aws
type: mvn
target: provider/crs-catalog-aws/pom.xml
\ No newline at end of file
......@@ -41,4 +41,8 @@ testing/*/venv/
testing/*/env/
#.pyc files in all directories
**/*.pyc
\ No newline at end of file
**/*.pyc
.DS_STORE
dist/
\ No newline at end of file
variables:
AWS_BUILD_SUBDIR: provider/crs-catalog-aws/build-aws
AWS_TEST_SUBDIR: testing/catalog_test_aws
AWS_SERVICE: crs-catalog
AWS_ENVIRONMENT: dev
AWS_INT_TEST_TYPE: python
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
......@@ -14,3 +21,6 @@ include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'scanners/gitlab-ultimate.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws.yml"
......@@ -21,6 +21,7 @@ The following software have components provided under the terms of this license:
- ASM Analysis (from )
- ASM Commons (from )
- ASM Core (from )
- ASM Core (from )
- ASM Tree (from )
- ASM Util (from )
- ASM based accessors helper used by json-smart (from )
......@@ -72,6 +73,7 @@ The following software have components provided under the terms of this license:
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
- Google HTTP Client Library for Java (from https://github.com/google/google-http-java-client.git)
- Gson (from https://github.com/google/gson)
- Gson (from https://github.com/google/gson)
- Guava InternalFutureFailureAccess and InternalFutures (from )
- Guava ListenableFuture only (from )
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
......@@ -178,6 +180,9 @@ The following software have components provided under the terms of this license:
- Spring AOP (from https://github.com/spring-projects/spring-framework)
- Spring Beans (from https://github.com/spring-projects/spring-framework)
- Spring Boot (from http://projects.spring.io/spring-boot/)
- Spring Boot Actuator (from http://projects.spring.io/spring-boot/)
- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure)
- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/)
- Spring Boot Jetty Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json)
......@@ -211,6 +216,7 @@ The following software have components provided under the terms of this license:
- javax.inject (from http://code.google.com/p/atinject/)
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
......@@ -247,9 +253,13 @@ BSD-2-Clause
========================================================================
The following software have components provided under the terms of this license:
- ASM Analysis (from )
- ASM Commons (from )
- ASM Tree (from )
- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- Lucene Common Analyzers (from )
- Plexus :: Default Container (from )
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
......@@ -262,13 +272,18 @@ BSD-3-Clause
The following software have components provided under the terms of this license:
- ASM Analysis (from )
- ASM Analysis (from )
- ASM Commons (from )
- ASM Commons (from )
- ASM Core (from )
- ASM Core (from )
- ASM Tree (from )
- ASM Tree (from )
- ASM Util (from )
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- JDOM (from http://www.jdom.org)
- JSch (from http://www.jcraft.com/jsch/)
- JavaBeans Activation Framework API jar (from )
......@@ -456,6 +471,7 @@ The following software have components provided under the terms of this license:
- SLF4J API Module (from http://www.slf4j.org)
- adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java)
- jwks-rsa (from http://www.auth0.com)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- mockito-junit-jupiter (from https://github.com/mockito/mockito)
- spring-security-core (from http://spring.io/spring-security)
......@@ -491,6 +507,7 @@ The following software have components provided under the terms of this license:
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- JTidy (from http://jtidy.sourceforge.net)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- xml-apis (from )
......@@ -531,7 +548,9 @@ The following software have components provided under the terms of this license:
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- Joda-Time (from http://www.joda.org/joda-time/)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Project Lombok (from https://projectlombok.org)
- Spring Web (from https://github.com/spring-projects/spring-framework)
......
......@@ -40,6 +40,7 @@
<module>provider/crs-catalog-azure/crs-catalog-aks</module>
<module>provider/crs-catalog-gcp/crs-catalog-gke</module>
<module>provider/crs-catalog-ibm/crs-catalog-ocp</module>
<module>provider/crs-catalog-aws</module>
</modules>
<repositories>
......
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
FROM amazoncorretto:8
ARG JAR_FILE=provider/crs-catalog-aws/target/crs-catalog-aws-*.jar
WORKDIR /
COPY ${JAR_FILE} app.jar
COPY /data/crs_catalog_v2.json /data/crs_catalog_v2.json
EXPOSE 8080
ENTRYPOINT java $JAVA_OPTS -jar /app.jar
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import boto3
import json
import os
import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
# env - CODEBUILD_RESOLVED_SOURCE_VERSION
parser.add_argument("--commit", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
parser.add_argument("--outdir", type=str, help="")
# full ecr image and tag, and any other artifacts
parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
commitId = args.commit
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
outputDir = args.outdir
artifacts = args.artifact
buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
commitArgs = {
"repositoryName": repoName,
"commitId": commitId
}
commitDetail = {
"commit": ""
}
# get the commit detail
try:
codecommit = boto3.client("codecommit")
commitDetail = codecommit.get_commit(**commitArgs)
except Exception as e:
print("Getting commit information from codecommit failed")
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
"commit": commitDetail["commit"]
}
print(json.dumps(buildInfo, sort_keys=True, indent=4))
# write the build.json file to dist
f = open(buildInfoFilePath, "w")
f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
f.close()
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
version: 0.2
phases:
install:
runtime-versions:
java: corretto8
commands:
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
- apt-get install -y maven
- java -version
- mvn -version
- mkdir -p /root/.m2
- cp ./provider/crs-catalog-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- export AWS_ACCOUNT_ID=`aws sts get-caller-identity | grep Account | cut -d':' -f 2 | cut -d'"' -f 2`
- export AWS_OSDU_DEV_MAVEN_AUTH_TOKEN=`aws codeartifact get-authorization-token --domain $AWS_OSDU_DEV_MAVEN_DOMAIN --domain-owner $AWS_ACCOUNT_ID --query authorizationToken --output text`
pre_build:
commands:
- echo "Logging in to Amazon ECR..."
- $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI
build:
commands:
- export REPO_NAME=${PWD##*/}
- export OUTPUT_DIR="dist"
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
- export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration
- export INTEGRATION_TEST_OUTPUT_BIN=${INTEGRATION_TEST_OUTPUT}/bin
- mkdir -p ${OUTPUT_DIR}/bin
- mkdir -p ${OUTPUT_DIR}/testing && mkdir -p ${INTEGRATION_TEST_OUTPUT} && mkdir -p ${INTEGRATION_TEST_OUTPUT}/bin
- echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
- printenv
- echo "Building primary service assemblies..."
- mvn -B test install -pl crs-catalog-core,provider/crs-catalog-aws -Ddeployment.environment=prod
- echo "Building integration testing assemblies and gathering artifacts..."
- ./testing/catalog_test_aws/build-aws/prepare-dist.sh
- echo "Building docker image..."
- docker build -f provider/crs-catalog-aws/build-aws/Dockerfile -t ${ECR_IMAGE} .
- docker tag ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST}
- echo "Pushing docker image..."
- docker push ${ECR_IMAGE}
- docker push ${ECR_IMAGE_BRANCH_LATEST}
- echo "Generate build-info.json"
- |
python provider/crs-catalog-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
reports:
SurefireReports: # CodeBuild will create a report group called "SurefireReports".
files: #Store all of the files
- "crs-catalog-core/target/surefire-reports/**/*"
- "provider/crs-catalog-aws/target/surefire-reports/**/*"
base-directory: "." # Location of the reports
artifacts:
files:
- "**/*"
base-directory: "dist"
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
cache:
paths:
- "/root/.m2/**/*"
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<profiles>
<profile>
<id>aws-osdu-dev-maven</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<repositories>
<repository>
<id>aws-osdu-dev-maven</id>
<url>${env.AWS_OSDU_DEV_MAVEN_URL}</url>
</repository>
<repository>
<id>gitlab-os-core-common-maven</id>
<url>https://community.opengroup.org/api/v4/projects/67/packages/maven</url>
</repository>
</repositories>
</profile>
<profile>
<id>credentialsConfiguration</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<deployment.environment>dev</deployment.environment>
<aws.accessKeyId>no-default</aws.accessKeyId>
<aws.secretKey>no-default</aws.secretKey>
<azure.devops.username>Another-Access-Token-2021</azure.devops.username>
<azure.devops.token>no-default</azure.devops.token>
</properties>
</profile>
</profiles>
<servers>
<server>
<id>aws-osdu-dev-maven</id>
<username>aws</username>
<password>${env.AWS_OSDU_DEV_MAVEN_AUTH_TOKEN}</password>
</server>
</servers>
<!-- CodeArtifact doesn't support external repos yet that aren't Maven Central. ETA Q4 2020. -->
<!-- <mirrors> -->
<!-- <mirror> -->
<!-- <id>aws-osdu-dev-maven</id> -->
<!-- <name>aws-osdu-dev-maven</name> -->
<!-- <url>https://osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url> -->
<!-- <mirrorOf>*,!gitlab-os-core-common-maven</mirrorOf> -->
<!-- </mirror> -->
<!-- </mirrors> -->
<activeProfiles>
<activeProfile>credentialsConfiguration</activeProfile>
</activeProfiles>
</settings>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu.crs-catalog-service</groupId>
<artifactId>crs-catalog-service</artifactId>
<version>1.0.0</version>
</parent>
<groupId>org.opengroup.osdu.crs-catalog-service</groupId>
<artifactId>crs-catalog-aws</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<name>crs-catalog-aws</name>
<description>CRS Catalog service on AWS</description>
<properties>
<app.version>1</app.version>
<app.id>crs-catalog-aws</app.id>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<objectify.version>5.1.22</objectify.version>
<slf4j.version>1.7.25</slf4j.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<prerequisites>
<maven>3.1.0</maven>
</prerequisites>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu.crs-catalog-service</groupId>
<artifactId>crs-catalog-core</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
</dependencies>
<build>
<!-- for hot reload of the web application-->
<outputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</outputDirectory>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<version>3.7.0</version>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.21.0</version>
<configuration>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/21/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/21/packages/maven</url>
</snapshotRepository>
</distributionManagement>
</project>
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.crs;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class CrsApplicationAWS {
public static void main(String[] args) {
SpringApplication.run(CrsApplicationAWS.class, args);
}
}
\ No newline at end of file
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.crs;
import org.opengroup.osdu.crs.model.CatalogImpl;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.io.FileReader;
@Configuration
public class CrsConfigurationAWS {
@Bean
public CatalogImpl getCatalog(@Value("${osdu.crs.catalog.filename}") String location) throws Exception {
try (FileReader reader = new FileReader(location)) {
return CatalogImpl.createCatalog(reader);
}
}
}
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.crs.security;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.opengroup.osdu.crs.middleware.AuthenticationRequestFilter;