Commit a0f41d57 authored by snehal jagtap's avatar snehal jagtap
Browse files

initial commit

parent a3069c56
Pipeline #61999 failed with stages
in 1 minute and 49 seconds
# osdu-writeback-publisher
# Introduction
TODO: Give a short introduction of your project. Let this section explain the objectives or the motivation behind this project.
# Getting Started
TODO: Guide users through getting your code up and running on their own system. In this section you can talk about:
1. Installation process
2. Software dependencies
3. Latest releases
4. API references
# Build and Test
TODO: Describe and show how to build your code and run the tests.
# Contribute
TODO: Explain how other users and developers can contribute to make your code better.
If you want to learn more about creating good readme files then refer the following [guidelines](https://docs.microsoft.com/en-us/azure/devops/repos/git/create-a-readme?view=azure-devops). You can also seek inspiration from the below readme files:
- [ASP.NET Core](https://github.com/aspnet/Home)
- [Visual Studio Code](https://github.com/Microsoft/vscode)
- [Chakra Core](https://github.com/Microsoft/ChakraCore)
\ No newline at end of file
name: $(Date:yyyyMMdd)$(Rev:.r)
trigger:
- master
- feature/*
variables:
- name: splitPipelineEnabled
value: false
resources:
repositories:
- repository: templates
type: git
name: azure-pipeline-templates
ref: master
- repository: variables
type: git
name: vsts-release-management
ref: master
pipelines:
- pipeline: Automation-Tests
project: PDI
source: Automation-Tests
branch: master
- pipeline: vsts-testresult-uploader
project: PDI
source: vsts-testresult-uploader
branch: master
- pipeline: pds-restassured-tests
project: PDI
source: pds-restassured-tests
branch: master
- pipeline: perf-tests
project: PDI
source: perf-tests
branch: master
stages:
- template: build-stage.yml@templates
parameters:
name: osdu-writeback-publisher
- ${{ if eq( variables['splitPipelineEnabled'], 'false' ) }}:
- template: run-deploy-stage-list.yml@templates
parameters:
name: osdu-writeback-publisher
restAssuredSmokeTest:
enabled: true
tagName: SmokeTestsWriteBackService
restAssuredSanityTest:
enabled: true
tagName: WitebackSanityTest
smokeTest:
enabled: false
testXMLFile: PDISmokeTest-WRITE-BACK-SERVICE.xml
sanityTest:
enabled: false
testXMLFile: PPS-SanityTest-WriteBackAPI.xml
regressionTest:
enabled: false
- ${{ if eq( variables['splitPipelineEnabled'], 'true' ) }}:
- template: run-deploy-stage-list-sp.yml@templates
parameters:
name: osdu-writeback-publisher
restAssuredSmokeTest:
enabled: true
tagName: SmokeTestsWriteBackService
restAssuredSanityTest:
enabled: true
tagName: WitebackSanityTest
smokeTest:
enabled: false
testXMLFile: PDISmokeTest-WRITE-BACK-SERVICE.xml
sanityTest:
enabled: false
testXMLFile: PPS-SanityTest-WriteBackAPI.xml
regressionTest:
enabled: false
\ No newline at end of file
import org.opengroup.osdu.production.dependencies.CommanCompilerFlags._
import org.opengroup.osdu.production.dependencies.CommonLibs._
import sbt.Keys._
import sbt._
lazy val scalaVer = "2.12.4"
lazy val osduOrg = "org.opengroup.osdu.production"
lazy val ItTest = config("it") extend Test
fork in IntegrationTest := true
parallelExecution in IntegrationTest := false
coverageExcludedPackages := "org.opengroup.osdu.production.domain.generated"
coverageMinimum := 90
coverageFailOnMinimum := true
scalacOptions ++= org.opengroup.osdu.production.dependencies.CommanCompilerFlags.scalaOptions
scalacOptions --= Seq("-Ywarn-dead-code")
lazy val core =
Project("osdu-writeback-publisher", file("."))
.configs(IntegrationTest)
.configs(ItTest)
.settings(inConfig(ItTest)(Defaults.testSettings) : _*)
.settings(
organization := osduOrg,
scalaVersion := scalaVer,
javacOptions ++= javac18,
fork in Test := true,
resolvers ++= Seq(GCSResolver.forBucket("pdi-ivy-snapshots-repo"), GCSResolver.forBucket("pdi-ivy-release-repo")),
libraryDependencies ++= Seq(
typeSafeconfig,
scalactic,
scalaCheck,
scalaLogging,
scalaTest,
akkaActor,
akkaTestKit,
akkaSlf4j,
akkaStream,
json4sNative,
scoverage,
mockito,
akkaQuartzScheduler exclude("org.quartz-scheduler" , "quartz") exclude("com.mchange" , "c3p0"),
"org.quartz-scheduler" % "quartz" % "2.3.2",
"com.mchange" % "c3p0" % "0.9.5.4",
"com.google.protobuf" % "protobuf-java" % "3.12.0",
"org.opengroup.osdu.production" %% "osdu-scala-commons" % "0.3.36" exclude("org.apache.httpcomponents", "httpclient"),
"org.opengroup.osdu.production" %% "osdu-akka-stream-commons" % "0.2.26" exclude("org.apache.httpcomponents", "httpclient"),
"org.opengroup.osdu.production" %% "osdu-security-commons" % "0.1.40" exclude ("com.google.oauth-client", "google-oauth-client-1.25.0") exclude ("com.google.oauth-client", "google-oauth-client-1.27.0") exclude("org.apache.httpcomponents", "httpclient")
).map(_.exclude("org.slf4j", "slf4j-log4j12"))
.map(_.exclude("com.google.guava","guava-20.0"))
.map(_.exclude("com.google.oauth-client", "google-oauth-client"))
).enablePlugins(JavaAppPackaging)
.enablePlugins(UniversalPlugin)
name in Universal := name.value
packageName in Universal := packageName.value
mainClass in Compile := Some("org.opengroup.osdu.production.App")
publishArtifact in(Compile, packageDoc) := false
publishArtifact in(Universal, packageDoc) := false
publishArtifact in(Universal, packageSrc) := false
lazy val ClasspathPattern = "declare -r app_classpath=\"(.*)\"\n".r
bashScriptDefines := bashScriptDefines.value.map {
case ClasspathPattern(classpath) => "declare -r app_classpath=\".:" + classpath + "\"\n"
case _@entry => entry
}
artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
artifact.name + "-" + module.revision + "-" + artifact.`type` + "." + artifact.extension
}
#!/bin/bash
declare -A envVars=( ["BUILD_BUILDNUMBER"]=$BUILD_BUILDNUMBER ["CONTAINER_REGISTRY_NAME"]=$CONTAINER_REGISTRY_NAME ["SONAR_URL"]=$SONAR_URL
["NEXUS_USERNAME"]=$NEXUS_USERNAME ["NEXUS_URL"]=$NEXUS_URL
["SONAR_LOGIN"]=$1 ["NEXUS_PASSWORD"]=$2 ["BINTRAY_PASSWORD"]=$3 )
declare -a missingEnvVars
for key in ${!envVars[@]}
do
if [[ -z ${envVars[${key}]} ]]
then
missingEnvVars+=( ${key} )
fi
done
if [ ${#missingEnvVars[@]} -gt 0 ]
then
joined=$(printf ", %s" "${missingEnvVars[@]}")
echo "Missing ${joined:1}"
exit 1
fi
export SONAR_LOGIN=$1
export NEXUS_PASSWORD=$2
export BINTRAY_PASSWORD=$3
sed -i -e "s/PROJECT_BUILD_VERSION/${BUILD_BUILDNUMBER}/g" sonar-project.properties
sed -i -e "s/GOOGLE_PROJECT_NAME/${GOOGLE_PROJECT_NAME}/g" ./src/main/resources/local.conf
sed -i -e "s/GLOBAL_GOOGLE_PROJECT_NAME/${GLOBAL_GOOGLE_PROJECT_NAME}/g" src/it/resources/it-test.conf
sed -i -e "s/GOOGLE_PROJECT_NAME/${GOOGLE_PROJECT_NAME}/g" src/it/resources/it-test.conf
sbt -sbt-launch-repo https://repo1.maven.org/maven2 clean coverage test it:test coverageReport coverageOff
#Fail if coverage has dropped
rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi
sbt universal:packageBin
if [ ! -f /usr/bin/sonar-scanner ]
then
wget https://dl.bintray.com/sonarsource/SonarQube/org/sonarsource/scanner/cli/sonar-scanner-cli/3.0.3.778/sonar-scanner-cli-3.0.3.778.zip && unzip sonar-scanner-cli-3.0.3.778.zip
cp -fR sonar-scanner-3.0.3.778/* /usr/
fi
sonar-scanner -Dsonar.host.url=${SONAR_URL} -Dsonar.login=${SONAR_LOGIN}
tar -cvzf coverage.tar.gz target/scala-2.12/scoverage-report
cp target/universal/osdu-writeback-publisher.zip build/osdu-writeback-publisher.zip
unzip build/osdu-writeback-publisher -d build
rm build/osdu-writeback-publisher.zip
cd build
docker build -t gcr.io/${CONTAINER_REGISTRY_NAME}/osdu-writeback-publisher:${BUILD_BUILDNUMBER} \
-t gcr.io/${CONTAINER_REGISTRY_NAME}/osdu-writeback-publisher:latest \
.
FROM openjdk:8u265-jre-slim
COPY osdu-writeback-publisher /opt/osdu-writeback-publisher
COPY run.sh /opt/osdu-writeback-publisher/bin
RUN cd /opt \
&& chmod +x -R /opt/osdu-writeback-publisher/bin \
&& apt-get update && apt-get install --only-upgrade bash && rm -rf /var/lib/apt/lists/*
WORKDIR /opt/osdu-writeback-publisher/bin
ENTRYPOINT [ "/opt/osdu-writeback-publisher/bin/run.sh", "app-conf" ]
\ No newline at end of file
#!/bin/sh
ln -s /opt/config-mount/app-config.conf /opt/osdu-writeback-publisher/bin/app-conf.conf
/opt/osdu-writeback-publisher/bin/osdu-writeback-publisher $1
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v1
appVersion: "1.0"
description: A Helm chart for Kubernetes
name: osdu-writeback-publisher
version: 0.1.0
cloud {
global-project {
name = "{{ .Values.globalResourcesProjectId }}"
}
pubsub {
writeback-data-publish-gateway{
subscription {
name = "writeback-data-publish-gateway-subscriber-{{ .Values.pds.envTag }}"
parallel-pull-count = 10
ack-deadline-seconds = 60
retain-acked-messages = true
}
topic {
name = "writeback-data-publish-gateway-{{ .Values.pds.envTag }}"
}
}
writeback-data-gateway{
subscription {
name = "writeback-data-gateway-subscriber-{{ .Values.pds.envTag }}"
parallel-pull-count = 10
ack-deadline-seconds = 60
retain-acked-messages = true
}
topic {
name = "writeback-data-gateway-{{ .Values.pds.envTag }}"
}
}
timeseries-data-gateway {
topic {
name = "{{ .Values.pds.envTag }}-data-gateway"
}
}
max-message-size = {{ .Values.pds.pubSubMaxMessageSizeInBytes }} // 8 MB
}
}
batch {
maxRetries = 10
delay = 2
batchSize = 100
maxParallel=10
}
dependent-services{
source-reference-service {
hostname = "source-reference-service",
port = 9090,
secret = "",
uri {
createStream = "/reference/api/v1/streams"
}
}
pdm-service {
hostname = "pdm-service",
port = 9090,
secret = "",
uri {
getPropertyByPropDesc = "/model/api/v3/properties/PROP_DESC"
}
}
structure-query-service {
hostname = "structure-query-service",
port = 9090,
secret = "",
uri {
getXrefFromUUID = "/structure/api/v1/getXrefsFromUUID/%s",
getElementFromXref = "/structure/api/v1/elements/getElementByXref?xref=%s"
}
}
structure-storage-service {
hostname="structure-service",
port=9090,
secret = "",
uri {
linkStream ="/metadata/api/v1/elements/%s/measurements"
}
}
}
http{
queue{
size=20
}
}
quartz {
schedules {
probeSchedule {
description = "A cron job that runs every minute",
expression = "0 * * ? * * *"
}
}
}
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "osdu-writeback-publisher.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "osdu-writeback-publisher.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "osdu-writeback-publisher.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Common labels
*/}}
{{- define "osdu-writeback-publisher.labels" -}}
app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
helm.sh/chart: {{ include "osdu-writeback-publisher.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "osdu-writeback-publisher.fullname" . }}-config
data:
app-config.conf: |
{{ tpl (.Files.Get "files/env.conf") . | indent 4 }}
\ No newline at end of file
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "osdu-writeback-publisher.fullname" . }}
labels:
{{ include "osdu-writeback-publisher.labels" . | indent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
labels:
app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.pds.imageRepository }}/{{ .Chart.Name }}:{{ .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
volumeMounts:
- name: application-config
mountPath: /opt/config-mount
- name: secret-config
mountPath: /opt/secret-mount
livenessProbe:
exec:
command:
- rm
- /opt/osdu-writeback-publisher/bin/alive
initialDelaySeconds: 90
periodSeconds: 120
failureThreshold: 3
resources:
{{ toYaml .Values.resources | indent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
volumes:
- name: ssl-certs
hostPath:
path: /etc/ssl/certs
- name: application-config
configMap:
name: {{ template "osdu-writeback-publisher.fullname" . }}-config
- name: secret-config
secret:
secretName: {{ template "osdu-writeback-publisher.fullname" . }}-secret
\ No newline at end of file
apiVersion: v1
kind: Secret
metadata:
name: {{ template "osdu-writeback-publisher.fullname" . }}-secret
type: Opaque
apiVersion: v1
kind: Service
metadata:
name: {{ template "osdu-writeback-publisher.fullname" . }}
labels:
{{ include "osdu-writeback-publisher.labels" . | indent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
\ No newline at end of file
replicaCount: 1
image:
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 9090
resources:
requests:
memory: 512Mi
cpu: 10m
limits:
memory: 2Gi
cpu: 100m
nodeSelector: {}
tolerations: []
affinity: {}
\ No newline at end of file
sbt.version=1.2.8
\ No newline at end of file
//format: OFF
resolvers += Classpaths.typesafeReleases
resolvers += Resolver.url("bintray-sbt-plugin-releases",url("https://dl.bintray.com/content/sbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
resolvers += GCSResolver.forBucket("pdi-ivy-release-repo")
val typesafeRepo = "com.typesafe.sbt"
addSbtPlugin(typesafeRepo % "sbt-git" % "0.9.3")
addSbtPlugin(typesafeRepo % "sbt-license-report" % "1.2.0")
addSbtPlugin(typesafeRepo % "sbt-native-packager" % "1.3.2")
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.8.0")
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.7")
addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2")
addSbtPlugin("net.vonbuchholtz" % "sbt-dependency-check" % "2.0.0")
addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.12")
addSbtPlugin("com.lightbend" % "sbt-google-cloud-storage" % "0.0.10")
addSbtPlugin("org.opengroup.osdu.production" % "osdu-common-deps" % "0.0.57")
import sbt.{Resolver, addSbtPlugin}
//format: OFF
addSbtPlugin("com.lightbend" % "sbt-google-cloud-storage" % "0.0.10")
\ No newline at end of file
## Description
Add your Code change description here.
## Type of changes is acceptable ( New Feature, Fix, Breaking Change )
- [ ] New Feature
- [ ] Fix
- [ ] Breaking Change
## Unit Test added in the build process and results are..
- [ ] Success
- [ ] Failed
- [ ] Not Applicable
## Integration Test added in the build process and results are..
- [ ] Success
- [ ] Failed
- [ ] Not Applicable