Skip to content
Snippets Groups Projects
Commit 698135d4 authored by David Diederich's avatar David Diederich
Browse files

Initial Import

parents
No related branches found
No related tags found
No related merge requests found
Showing
with 1285 additions and 0 deletions
# Compiled class file
*.class
# Log file
*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
*.nar
*.ear
*.zip
*.tar.gz
*.rar
### STS ###
target/
.settings/
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
## Running the WKS service locally
The WKS Service is a Maven multi-module project with each cloud implemention placed in its submodule. WKS service listen to records_change event of Storage Service. To build or run Schema Service locally, follow the below steps :
1. Clone the os-wks repository from git . Below is the URL :
[https://dev.azure.com/slb-swt/data-management/_git/os-wks](https://dev.azure.com/slb-swt/data-management/_git/os-wks)
2. Navigate to the root of the wks project, os-wks. For building the project using command line, run below command :
```bash
mvn clean install
```
This will build the core project as well as all the underlying projects. If we want to build projects for specific cloud vendor, we can use mvn --projects command. For example, if we want to build only for GCP(Google Cloud Platform), we can use below command :
```bash
mvn --projects wks-core,provider/wks-gcp clean install
```
3. Run wks service in command line. We need to select which cloud vendor specific wks-service we want to run. For example, if we want to run wks-service for GCP, run the below command :
```bash
# Running GCP :
java -jar -Dspring.profiles.active=default provider\wks-gcp\target\os-wks-gcp-0.0.1-spring-boot.jar
## License
Copyright 2017-2020, Schlumberger
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License
\ No newline at end of file
trigger:
branches:
include:
- master
paths:
exclude:
- README.md
- .gitignore
variables:
- group: MAVEN_BUILD_VARS
- name: osProjectName
value: wks
- name: dockerImageName
value: os-$(osProjectName)-app
- name: tag
value: $(Build.BuildNumber)
- name: dockerDir
value: provider/$(osProjectName)-gcp/docker
- name: deploymentName
value: os-$(osProjectName)-service
stages:
- stage: Build
jobs:
- job: Build
pool:
name: Hosted Ubuntu 1604
demands: Maven
steps:
- task: DownloadSecureFile@1
name: gcrKey
inputs:
secureFile: gcr-push-key-file.json
- task: DownloadSecureFile@1
name: kubeconfig
inputs:
secureFile: 'kubeconfig'
- task: Maven@3
displayName: Maven Build
inputs:
mavenPomFile: 'pom.xml'
mavenOptions: '-Xmx3072m'
javaHomeOption: 'JDKVersion'
jdkVersionOption: '1.8'
jdkArchitectureOption: 'x64'
publishJUnitResults: true
testResultsFiles: '**/surefire-reports/TEST-*.xml'
options: '--settings maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
goals: 'clean install package'
- task: CopyFiles@2
displayName: 'Copy Artifacts to: $(build.artifactstagingdirectory)/os-$(osProjectName)'
inputs:
Contents: |
provider/$(osProjectName)-gcp/deployments/*
provider/$(osProjectName)-gcp/target/*-spring-boot.jar
TargetFolder: '$(build.artifactstagingdirectory)/os-$(osProjectName)'
flattenFolders: true
condition: succeeded()
- task: Bash@3
inputs:
targetType: 'inline'
script: |
#!/bin/bash
pushd $(dockerDir)
cat $(gcrKey.secureFilePath) | docker login -u _json_key --password-stdin https://gcr.io
echo $(dockerImageName)
docker-compose build $(dockerImageName)
docker tag gcr.io/opendes/$(dockerImageName) gcr.io/opendes/$(dockerImageName):$(tag)
docker push gcr.io/opendes/$(dockerImageName):$(tag)
docker push gcr.io/opendes/$(dockerImageName)
echo 'Push done.'
kubectl --kubeconfig $(kubeconfig.secureFilePath) rollout restart deployment/$(deploymentName)
popd
sleep 10
OUTPUT="{\"status\":\"UP\"}"
ENDPOINT=$(WKS_SERVICE_HEALTH_URL)
echo $ENDPOINT
while [ -z "$STATUS" ]; do
STATUS=`curl -v --silent --http1.0 "$ENDPOINT" 2>&1 | grep "$OUTPUT"`
echo $STATUS
if [ -z "$STATUS" ]; then
echo "Endpoint is not up yet."
sleep 10
else
echo "Endpoint is up"
fi
done
condition: succeeded()
displayName: 'build,upload and deploy docker image'
- task: Maven@3
displayName: 'Running IntegrationTest'
inputs:
mavenPomFile: '$(osProjectName)-core/pom.xml'
goals: 'verify'
options: '-P IntegrationTest --settings maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
publishJUnitResults: false
javaHomeOption: 'JDKVersion'
mavenVersionOption: 'Default'
mavenAuthenticateFeed: false
effectivePomSkip: false
sonarQubeRunAnalysis: false
env:
INTEGRATION_TEST_AUDIENCE: $(INTEGRATION_TEST_AUDIENCE)
INTEGRATION_TESTER : $(INTEGRATION_TESTER)
WKS_BUCKET_NAME : $(WKS_BUCKET_NAME)
STORAGE_SERVICE_URL : $(STORAGE_SERVICE_URL)
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: drop'
inputs:
PathtoPublish: '$(build.artifactstagingdirectory)/os-$(osProjectName)'
ArtifactName: 'drop'
publishLocation: 'Container'
condition: succeededOrFailed()
- stage: DeployToQA
condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
variables:
sourceImageName: gcr.io/opendes/$(dockerImageName)
destinationImageName: us.gcr.io/opendes-evt/$(dockerImageName)
jobs:
- job: DeployToQA
steps:
- task: DownloadSecureFile@1
name: gcrKey
inputs:
secureFile: cicd-push-image-to-cr-keyfile.json
- task: DownloadSecureFile@1
name: gcrKeyEvt
inputs:
secureFile: cicd-push-image-to-cr-evt-keyfile.json
- task: DownloadSecureFile@1
name: kuberConfigEvt
inputs:
secureFile: kubeconfig-evt-opendes-qa-us
- bash: |
#!/bin/bash
set -e
cat $(gcrKey.secureFilePath) | docker login -u _json_key --password-stdin https://gcr.io
docker pull $(sourceImageName):$(tag)
cat $(gcrKeyEvt.secureFilePath) | docker login -u _json_key --password-stdin https://us.gcr.io
docker tag $(sourceImageName):$(tag) $(destinationImageName):$(tag)
docker tag $(sourceImageName):$(tag) $(destinationImageName)
docker push $(destinationImageName):$(tag)
docker push $(destinationImageName)
kubectl --kubeconfig $(kuberConfigEvt.secureFilePath) rollout restart deployment/$(deploymentName)
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>os-core</id>
<username>slb-des-ext-collaboration</username>
<!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
<!-- The generated token expires on or before 11/14/2019 -->
<password>${VSTS_FEED_TOKEN}</password>
</server>
</servers>
</settings>
\ No newline at end of file
mvnw 0 → 100644
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven2 Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -f /etc/mavenrc ] ; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="`/usr/libexec/java_home`"
else
export JAVA_HOME="/Library/Java/Home"
fi
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
fi
fi
if [ -z "$M2_HOME" ] ; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
fi
done
saveddir=`pwd`
M2_HOME=`dirname "$PRG"`/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
# For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw ; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
# TODO classpath?
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`which java`"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]
then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd`
fi
# end of workaround
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
fi
}
BASE_DIR=`find_maven_basedir "$(pwd)"`
if [ -z "$BASE_DIR" ]; then
exit 1;
fi
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR
fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
fi
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
exec "$JAVACMD" \
$MAVEN_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
mvnw.cmd 0 → 100644
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Maven2 Start Up Batch script
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM M2_HOME - location of maven2's installed home dir
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%" == "on" pause
if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
exit /B %ERROR_CODE%
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<properties>
<java.version>1.8</java.version>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<springfox-version>2.7.0</springfox-version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>2.2.6.RELEASE</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-wks</artifactId>
<version>0.0.1</version>
<packaging>pom</packaging>
<name>os-wks</name>
<description>os wks service </description>
<modules>
<module>wks-core</module>
<module>provider/wks-gcp</module>
</modules>
<repositories>
<repository>
<id>os-core</id>
<url>https://pkgs.dev.azure.com/slb-des-ext-collaboration/_packaging/os-core/maven/v1</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</project>
FROM openjdk:8-slim
RUN apt-get update && apt-get install -y curl
ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","-Dspring.profiles.active=kuber","/app.jar"]
EXPOSE 8080
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
version: "3"
services:
os-wks-app:
build:
args:
JAR_FILE: target/os-wks-gcp-0.0.1-spring-boot.jar
context: ..
dockerfile: docker/Dockerfile
image: gcr.io/opendes/os-wks-app
ports:
- "8080:8080"
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-wks</artifactId>
<version>0.0.1</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>os-wks-gcp</artifactId>
<description>Google cloud related implementation stuff.</description>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-wks-core</artifactId>
<version>0.0.1</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.0.11</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.1.17</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>1.105.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-datastore</artifactId>
<version>1.102.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-iam</artifactId>
<version>v1-rev310-1.25.0</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.1.15</version>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java8</artifactId>
<version>5.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit</artifactId>
<version>5.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-guice</artifactId>
<version>5.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>4.2.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
org.opengroup.osdu.wks.WksServiceApplication
</mainClass>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<configuration>
<failOnMissingWebXml>false</failOnMissingWebXml>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M3</version>
<configuration>
<skipTests>${skipUnitTests}</skipTests>
<forkCount>3</forkCount>
<reuseForks>true</reuseForks>
<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package org.opengroup.osdu.wks;
import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class GcpConfiguration {
@Bean
public TenantFactory getTenantFactory() {
return new TenantFactory();
}
}
package org.opengroup.osdu.wks.constants;
public class Constants {
public static final String SUBSCRIPTION_ID = "records-changed-sub-%s-wks";
public static final String BEARER= "Bearer ";
public static final String DATA_PARTITION_ID = "data-partition-id";
public static final String CORRELATION_ID = "correlation-id";
public static final String MAPPING_NOT_PRESENT = "Mapping not present";
public static final String MAPPING_MULTIPLE_FILE_FOUND = "Expected single mapping file";
public static final String COLON_SEPARATOR = ":";
}
package org.opengroup.osdu.wks.credentials;
import java.io.IOException;
import java.security.GeneralSecurityException;
import org.springframework.stereotype.Component;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpTransport;
@Component
public class GoogleIamHelper {
public HttpTransport retrieveHttpTransport() throws GeneralSecurityException, IOException {
return GoogleNetHttpTransport.newTrustedTransport();
}
public GoogleCredential retriveCredential() throws IOException {
return GoogleCredential.getApplicationDefault();
}
}
package org.opengroup.osdu.wks.credentials;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.iam.v1.Iam;
import com.google.api.services.iam.v1.IamScopes;
import com.google.api.services.iam.v1.model.SignJwtRequest;
import com.google.api.services.iam.v1.model.SignJwtResponse;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import lombok.Setter;
import lombok.extern.java.Log;
import org.apache.http.HttpHeaders;
import org.apache.http.entity.ContentType;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.wks.constants.Constants;
import org.opengroup.osdu.wks.exceptions.ApplicationException;
import org.opengroup.osdu.wks.provider.interfaces.UserCredential;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.*;
import java.util.logging.Level;
@Log
@Component("userCredential")
public class JwtTokenGenerator implements UserCredential {
private final JsonFactory jsonFactory = new JacksonFactory();
private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s";
private static final String GOOGLE_ID_TOKEN_URL= "https://www.googleapis.com/oauth2/v4/token";
private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token";
private static final String GRANT_TYPE = "grant_type";
private static final String ASSERTION = "assertion";
private static final String URN_OAUTH_JWT_BEARER = "urn:ietf:params:oauth:grant-type:jwt-bearer";
@Setter
private Iam iam;
@Autowired
private RestTemplate restTemplate;
@Autowired
private GoogleIamHelper googleIamHelper;
@Value("${GOOGLE_AUDIENCES}")
private String targetAudience;
public String getIdToken(TenantInfo tenant) throws ApplicationException {
try {
String signedJwt = getSignedJWT(tenant);
JsonObject jsonContent = postToUrl(signedJwt);
if (!jsonContent.has("id_token")) {
log.log(Level.SEVERE, String.format("Google IAM response: %s", jsonContent.toString()));
throw new ApplicationException("Authorization Failed");
}
return Constants.BEARER+jsonContent.get("id_token").getAsString();
} catch (ApplicationException | IOException | GeneralSecurityException e) {
throw new ApplicationException("Error generating token", e);
}
}
private String getSignedJWT(TenantInfo tenant) throws IOException, GeneralSecurityException {
Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant);
SignJwtRequest signJwtRequest = new SignJwtRequest();
signJwtRequest.setPayload(jsonFactory.toString(signJwtPayload));
String serviceAccountName = String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(),
tenant.getServiceAccount());
Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts()
.signJwt(serviceAccountName, signJwtRequest);
SignJwtResponse signJwtResponse = signJwt.execute();
return signJwtResponse.getSignedJwt();
}
private JsonObject postToUrl(String signedJwt){
HttpEntity<Object> payload = populatePayload(signedJwt);
ResponseEntity<String> entity = restTemplate.postForEntity(GOOGLE_ID_TOKEN_URL, payload, String.class);
return new JsonParser().parse(Objects.requireNonNull(entity.getBody())).getAsJsonObject();
}
private HttpEntity<Object> populatePayload(String signedJwt) {
MultiValueMap<String, String> headers = new LinkedMultiValueMap<>();
headers.add(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType());
MultiValueMap<String, String> body = new LinkedMultiValueMap<>();
body.add(GRANT_TYPE, URN_OAUTH_JWT_BEARER);
body.add(ASSERTION, signedJwt);
return new HttpEntity<>(body, headers);
}
public Iam getIam() throws GeneralSecurityException, IOException {
if (this.iam == null) {
HttpTransport httpTransport = googleIamHelper.retrieveHttpTransport();
GoogleCredential credential = googleIamHelper.retriveCredential();
if (credential.createScopedRequired()) {
List<String> scopes = new ArrayList<>();
scopes.add(IamScopes.CLOUD_PLATFORM);
credential = credential.createScoped(scopes);
}
this.iam = new Iam.Builder(httpTransport, jsonFactory, credential).build();
}
return this.iam;
}
private Map<String, Object> getJWTCreationPayload(TenantInfo tenant) {
Map<String, Object> payload = new HashMap<>();
payload.put("target_audience", targetAudience);
payload.put("exp", System.currentTimeMillis() / 1000 + 3600);
payload.put("iat", System.currentTimeMillis() / 1000);
payload.put("iss", tenant.getServiceAccount());
payload.put("aud", JWT_AUDIENCE);
return payload;
}
}
package org.opengroup.osdu.wks.pubsub;
import java.util.Arrays;
import java.util.logging.Level;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.wks.constants.Constants;
import org.opengroup.osdu.wks.exceptions.ApplicationException;
import org.opengroup.osdu.wks.exceptions.BadRequestException;
import org.opengroup.osdu.wks.model.RawRecordDetails;
import org.opengroup.osdu.wks.service.WKSService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.google.cloud.pubsub.v1.AckReplyConsumer;
import com.google.cloud.pubsub.v1.MessageReceiver;
import com.google.gson.Gson;
import com.google.pubsub.v1.PubsubMessage;
import lombok.extern.java.Log;
@Service
@Log
public class PubSubMessageReceiver implements MessageReceiver {
@Autowired
private WKSService wKSService;
@Override
public void receiveMessage(PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) {
try {
String dataPartitionId = pubsubMessage.getAttributesMap().get(Constants.DATA_PARTITION_ID);
String correlationId = pubsubMessage.getAttributesMap().get(Constants.CORRELATION_ID);
processWksTransform(pubsubMessage, dataPartitionId, correlationId);
} catch (BadRequestException e) {
log.severe(e.getErrorMsg());
log.log(Level.SEVERE, String.format("Bad Request Reason: %s, pubsub message id: %s", e.getErrorMsg(),
pubsubMessage.getMessageId()));
} catch (ApplicationException e) {
log.severe(e.getErrorMsg());
log.log(Level.SEVERE, String.format("Application Error Reason: %s, pubsub message id: %s", e.getErrorMsg(),
pubsubMessage.getMessageId()));
}
ackReplyConsumer.ack();
}
private void processWksTransform(PubsubMessage pubsubMessage, String dataPartitionId, String correlationId)
throws BadRequestException, ApplicationException {
log.log(Level.INFO, "Transformation started for " + pubsubMessage.getMessageId());
RawRecordDetails[] rawRecordDetails = retriveDataFromPubSubMsg(pubsubMessage);
log.log(Level.INFO, Arrays.toString(rawRecordDetails));
wKSService.transform(rawRecordDetails,
dataPartitionId,
correlationId);
log.log(Level.INFO, "Transformation completed for" + pubsubMessage.getMessageId());
}
private RawRecordDetails[] retriveDataFromPubSubMsg(PubsubMessage msg) throws BadRequestException {
String messageData = msg.getData().toStringUtf8();
return getRecordsChangeData(messageData);
}
/**
* This method receives Base64 encoded pubsub data and returns decoded data as
* {@link RawRecordDetails}
*
* @param data
* @return {@link RawRecordDetails}
* @throws BadRequestException
*/
private RawRecordDetails[] getRecordsChangeData(String data) throws BadRequestException {
Gson gson = new Gson();
if (StringUtils.isBlank(data)) {
throw new BadRequestException("PubSub message not present");
}
return gson.fromJson(data, RawRecordDetails[].class);
}
}
package org.opengroup.osdu.wks.pubsub;
import org.springframework.stereotype.Component;
import com.google.api.core.ApiService.Listener;
import com.google.api.core.ApiService.State;
import lombok.extern.java.Log;
@Log
@Component
public class SubscriptionListener extends Listener {
@Override
public void failed(State from, Throwable failure) {
log.severe(failure.getStackTrace()[0].toString());
}
}
package org.opengroup.osdu.wks.pubsub;
import com.google.api.gax.batching.FlowControlSettings;
import com.google.cloud.pubsub.v1.Subscriber;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.pubsub.v1.ProjectSubscriptionName;
import lombok.Setter;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.wks.constants.Constants;
import org.opengroup.osdu.wks.provider.interfaces.SubscriptionManager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Collection;
@Component
public class SubscriptionManagerImpl implements SubscriptionManager {
@Autowired
private ITenantFactory tenantFactory;
@Autowired
private PubSubMessageReceiver pubSubMessageReceiver;
@Autowired
private SubscriptionListener subscriptionListener;
@Setter
private String projectID;
public SubscriptionManagerImpl(@Value("${GOOGLE_CLOUD_PROJECT}") String projectID){
this.projectID =projectID;
}
@Override
public void subscribeRecordsChangeEvent() {
Collection<TenantInfo> tenants = tenantFactory.listTenantInfo();
for (TenantInfo tenant : tenants)
subscribeTenant(tenant.getName());
}
private FlowControlSettings buildFlowControl() {
long elementCount = 5;
return FlowControlSettings.newBuilder().setMaxOutstandingElementCount(elementCount).build();
}
private void subscribeTenant(String tenantName) {
String projectId = projectID;
subscribeToTopic(projectId, tenantName);
}
private void subscribeToTopic(String projectId, String tenantName) {
String subscriptionId = String.format(Constants.SUBSCRIPTION_ID, tenantName);
subscribe(subscriptionId, projectId);
}
private void subscribe(String subscriptionId, String projectId) {
ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(projectId, subscriptionId);
Subscriber subscriber = Subscriber.newBuilder(subscriptionName, pubSubMessageReceiver)
.setFlowControlSettings(buildFlowControl()).build();
subscriber.addListener(subscriptionListener, MoreExecutors.directExecutor());
subscriber.startAsync();
}
}
package org.opengroup.osdu.wks.storage;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.gax.paging.Page;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobListOption;
import lombok.extern.java.Log;
import org.opengroup.osdu.wks.constants.Constants;
import org.opengroup.osdu.wks.model.MappingsModel;
import org.opengroup.osdu.wks.provider.interfaces.MappingStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* This class is to fetch Mapping file from Google storage bucket.
*
*/
@Log
@Service("mappingStore")
public class MappingStoreImpl implements MappingStore {
@Autowired
private StorageFactory storageFactory;
@Value("${GOOGLE_CLOUD_PROJECT}-wks")
private String bucketName;
/**
* This method accepts filePath to fetch mapping file from
* Google storage bucket
*
*/
@Override
public MappingsModel getMapping(final String fileName) {
MappingsModel mappings = null;
try {
Blob mappingContent = retrieveMappingContent( fileName);
mappings = convertBlobToMappings(mappingContent);
} catch (IOException e) {
log.warning( e.getMessage());
}
return mappings;
}
private MappingsModel convertBlobToMappings(Blob blob) throws IOException {
byte[] content = blob.getContent();
String jsonStr = new String(content);
ObjectMapper mapper = new ObjectMapper();
return mapper.readValue(jsonStr, MappingsModel.class);
}
private Blob retrieveMappingContent(String fileName) throws IOException {
Storage storage = storageFactory.retrieveStorage();
BlobListOption prefix = Storage.BlobListOption
.prefix(fileName);
BlobListOption currentDir = Storage.BlobListOption.currentDirectory();
Page<Blob> blobs = storage.list(this.bucketName, currentDir, prefix);
return retrieveSingleBlob(blobs);
}
private Blob retrieveSingleBlob(Page<Blob> blobs) throws IOException {
List<Blob> blobList = new ArrayList<>();
for (Blob blob : blobs.iterateAll()) {
blobList.add(blob);
}
if (blobList.size() > 1) {
throw new IOException(Constants.MAPPING_MULTIPLE_FILE_FOUND);
}
if (blobList.isEmpty()) {
throw new FileNotFoundException(Constants.MAPPING_NOT_PRESENT);
}
return blobList.get(0);
}
}
package org.opengroup.osdu.wks.storage;
import org.springframework.stereotype.Component;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
@Component
public class StorageFactory {
public Storage retrieveStorage() {
return StorageOptions.getDefaultInstance().getService();
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment