Commit 228eb597 authored by Brandt Beal's avatar Brandt Beal Committed by Rucha Deshpande
Browse files

feat: initial aws provider implementation

parent c0641226
......@@ -19,7 +19,9 @@
*.tar.gz
*.rar
# IntelliJ
.idea
*.iml
### STS ###
target/
......
......@@ -40,6 +40,7 @@
<modules>
<module>schema-core</module>
<module>provider/schema-aws</module>
<module>provider/schema-gcp</module>
<module>provider/schema-ibm</module>
<module>provider/schema-azure</module>
......
# Copyright © Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
FROM amazoncorretto:8
ARG JAR_FILE=provider/schema-aws/target/*-spring-boot.jar
WORKDIR /
COPY ${JAR_FILE} app.jar
EXPOSE 8080
ENTRYPOINT java $JAVA_OPTS -jar /app.jar
import boto3
import json
import os
import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
# env - CODEBUILD_RESOLVED_SOURCE_VERSION
parser.add_argument("--commit", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
parser.add_argument("--outdir", type=str, help="")
# full ecr image and tag, and any other artifacts
parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
commitId = args.commit
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
outputDir = args.outdir
artifacts = args.artifact
buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
commitArgs = {
"repositoryName": repoName,
"commitId": commitId
}
commitDetail = {
"commit": ""
}
# get the commit detail
try:
codecommit = boto3.client("codecommit")
commitDetail = codecommit.get_commit(**commitArgs)
except Exception as e:
print("Getting commit information from codecommit failed")
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
"commit": commitDetail["commit"]
}
print(json.dumps(buildInfo, sort_keys=True, indent=4))
# write the build.json file to dist
f = open(buildInfoFilePath, "w")
f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
f.close()
# Copyright © Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
version: 0.2
phases:
install:
runtime-versions:
java: openjdk8
commands:
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
- apt-get install -y maven
- java -version
- mvn -version
- mkdir -p /root/.m2
- aws s3 sync s3://$M2_REPO_S3_BUCKET /root/.m2 # copy previous state of the shared libraries' .m2 folder from S3 to local
pre_build:
commands:
- echo "Logging in to Amazon ECR..."
- $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI
build:
commands:
- export REPO_NAME=${PWD##*/}
- export OUTPUT_DIR="dist"
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration
- export INTEGRATION_TEST_OUTPUT_BIN=${INTEGRATION_TEST_OUTPUT}/bin
- mkdir -p ${OUTPUT_DIR}/bin
- mkdir -p ${OUTPUT_DIR}/testing && mkdir -p ${INTEGRATION_TEST_OUTPUT} && mkdir -p ${INTEGRATION_TEST_OUTPUT}/bin
- echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
- printenv
- echo "Building primary service assemblies..."
- mvn install -N # required for this service to install the parent pom so that the integration tests will find it
- mvn -B test install -pl schema-core,provider/schema-aws -Ddeployment.environment=prod
# Suspended until further notice
# - echo "Copying assemblies to dist..."
# - cp ./provider/delivery-aws/target/*spring-boot.jar ${OUTPUT_DIR}/bin # copy aws jars
# - cp ./delivery-core/target/*.jar ${OUTPUT_DIR}/bin # copy core jar
# - echo "Building integration testing assemblies and gathering artifacts..."
# - ./testing/schema-test-aws/build-aws/prepare-dist.sh
- echo "Building docker image..."
- docker build -f provider/schema-aws/build-aws/Dockerfile -t ${ECR_IMAGE} .
- echo "Pushing docker image..."
- docker push ${ECR_IMAGE}
- echo "Generate build-info.json"
- |
python provider/schema-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
reports:
SurefireReports: # CodeBuild will create a report group called "SurefireReports".
files: #Store all of the files
- "schema-core/target/surefire-reports/**/*"
- "provider/schema-aws/target/surefire-reports/**/*"
base-directory: "." # Location of the reports
artifacts:
files:
- "**/*"
base-directory: "dist"
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
cache:
paths:
- "/root/.m2/**/*"
\ No newline at end of file
{
"name": "schema-service",
"description": "Build of the os-delivery repository",
"source": {
"type": "CODECOMMIT",
"location": "https://git-codecommit.us-east-1.amazonaws.com/v1/repos/schema-service",
"gitCloneDepth": 1,
"gitSubmodulesConfig": {
"fetchSubmodules": false
},
"buildspec": "./provider/schema-aws/build-aws/buildspec.yaml",
"insecureSsl": false
},
"secondarySources": [],
"sourceVersion": "refs/heads/dev",
"secondarySourceVersions": [],
"artifacts": {
"type": "S3",
"location": "888733619319-devops-build-artifacts",
"path": "schema-service",
"namespaceType": "NONE",
"name": "schema-service",
"packaging": "ZIP",
"overrideArtifactName": true,
"encryptionDisabled": false
},
"secondaryArtifacts": [],
"cache": {
"type": "LOCAL",
"modes": [
"LOCAL_CUSTOM_CACHE"
]
},
"environment": {
"type": "LINUX_CONTAINER",
"image": "aws/codebuild/standard:2.0",
"computeType": "BUILD_GENERAL1_SMALL",
"environmentVariables": [
{
"name": "M2_REPO_S3_BUCKET",
"value": "dev-888733619319-persistent-maven-m2-bucket",
"type": "PLAINTEXT"
},
{
"name": "ECR_REGISTRY",
"value": "888733619319.dkr.ecr.us-east-1.amazonaws.com/schema-service_dev",
"type": "PLAINTEXT"
}
],
"privilegedMode": true,
"imagePullCredentialsType": "CODEBUILD"
},
"serviceRole": "arn:aws:iam::888733619319:role/service-role/dev-CodeBuildRole",
"timeoutInMinutes": 60,
"queuedTimeoutInMinutes": 480,
"encryptionKey": "arn:aws:kms:us-east-1:888733619319:alias/aws/s3",
"tags": [],
"vpcConfig": {
"vpcId": "vpc-0f273733df61bc541",
"subnets": [
"subnet-03963a50e77043e12",
"subnet-04a975f0e6e0c9279"
],
"securityGroupIds": [
"sg-0dee4e811c2062e26"
]
},
"badgeEnabled": true,
"logsConfig": {
"cloudWatchLogs": {
"status": "ENABLED"
},
"s3Logs": {
"status": "DISABLED",
"encryptionDisabled": false
}
}
}
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>os-schema</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.0.1</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>os-schema-aws</artifactId>
<description>AWS implementation of the OSDU Schema Service</description>
<packaging>jar</packaging>
<properties>
<aws.version>1.11.637</aws.version>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-core</artifactId>
<version>0.0.1</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java8</artifactId>
<version>5.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit</artifactId>
<version>5.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-guice</artifactId>
<version>5.4.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.1.9.RELEASE</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
org.opengroup.osdu.schema.SchemaApplication
</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/26/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/26/packages/maven</url>
</snapshotRepository>
</distributionManagement>
</project>
\ No newline at end of file
package org.opengroup.osdu.schema.provider.aws;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication(exclude = {
org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class})
@ComponentScan({"org.opengroup"})
public class SchemaAwsApplication {
public static void main(String[] args) {
SpringApplication.run(SchemaAwsApplication.class, args);
}
}
package org.opengroup.osdu.schema.provider.aws.provider.schemainfostore;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.BadRequestException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.opengroup.osdu.schema.model.Authority;
import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.IAuthorityStore;
import org.springframework.stereotype.Repository;
@Repository
public class AwsAuthorityStore implements IAuthorityStore {
@Override
public Authority get(String authorityId) throws NotFoundException, ApplicationException {
return null;
}
@Override
public Authority create(Authority authority) throws ApplicationException, BadRequestException {
return null;
}
}
package org.opengroup.osdu.schema.provider.aws.provider.schemainfostore;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.BadRequestException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.opengroup.osdu.schema.model.EntityType;
import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.IEntityTypeStore;
import org.springframework.stereotype.Repository;
@Repository
public class AwsEntityTypeStore implements IEntityTypeStore {
@Override
public EntityType get(String entityTypeId) throws NotFoundException, ApplicationException {
return null;
}
@Override
public EntityType create(EntityType entityType) throws BadRequestException, ApplicationException {
return null;
}
}
package org.opengroup.osdu.schema.provider.aws.provider.schemainfostore;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.BadRequestException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.opengroup.osdu.schema.model.QueryParams;
import org.opengroup.osdu.schema.model.SchemaInfo;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfoStore;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public class AwsSchemaInfoStore implements ISchemaInfoStore {
@Override
public SchemaInfo updateSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException {
return null;
}
@Override
public SchemaInfo createSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException {
return null;
}
@Override
public SchemaInfo getSchemaInfo(String schemaId) throws ApplicationException, NotFoundException {
return null;
}
@Override
public String getLatestMinorVerSchema(SchemaInfo schemaInfo) throws ApplicationException {
return null;
}
@Override
public List<SchemaInfo> getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException {
return null;
}
@Override
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
return false;
}
@Override
public boolean cleanSchema(String schemaId) throws ApplicationException {
return false;
}
}
package org.opengroup.osdu.schema.provider.aws.provider.schemainfostore;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.BadRequestException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.opengroup.osdu.schema.model.Source;
import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISourceStore;
import org.springframework.stereotype.Repository;
@Repository
public class AwsSourceStore implements ISourceStore {
@Override
public Source get(String sourceId) throws NotFoundException, ApplicationException {
return null;
}
@Override
public Source create(Source source) throws BadRequestException, ApplicationException {
return null;
}
}
package org.opengroup.osdu.schema.provider.aws.provider.schemastore;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore;
import org.springframework.stereotype.Repository;
@Repository
public class AwsSchemaStore implements ISchemaStore {
@Override
public String createSchema(String filePath, String content) throws ApplicationException {
return null;
}
@Override
public String getSchema(String dataPartitionId, String filePath) throws NotFoundException, ApplicationException {
return null;
}
@Override
public boolean cleanSchemaProject(String schemaId) throws ApplicationException {
return false;
}
}
server.servlet.context-path=/api/schema-service/v1
logging.level.org.springframework.web=${LOG_LEVEL:INFO}
server.port=${APPLICATION_PORT:8080}
aws.resource.prefix=${ENVIRONMENT}
aws.ssm=${SSM_ENABLED:True}
AUTHORIZE_API=${ENTITLEMENTS_URL:http://localhost:8081}/api/entitlements/v1
AUTHORIZE_API_KEY=
LOG_PREFIX=schema
## AWS DynamoDB configuration
# needed for tenantfactory
aws.dynamodb.key=kind
aws.dynamodb.table.prefix=${aws.resource.prefix}-
aws.region=${AWS_REGION}
aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
## AWS Lambda configuration
aws.lambda.get-groups-function-name=${aws.resource.prefix}-os-entitlements-GroupsFunction
management.health.elasticsearch.enabled=false
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<include resource="org/springframework/boot/logging/logback/base.xml" />
<logger name="org.springframework" level="WARN"/>
</configuration>