initial commit

README.md

-# osdu-writeback-publisher
+# Introduction 
+TODO: Give a short introduction of your project. Let this section explain the objectives or the motivation behind this project. 
 
+# Getting Started
+TODO: Guide users through getting your code up and running on their own system. In this section you can talk about:
+1.	Installation process
+2.	Software dependencies
+3.	Latest releases
+4.	API references
+
+# Build and Test
+TODO: Describe and show how to build your code and run the tests. 
+
+# Contribute
+TODO: Explain how other users and developers can contribute to make your code better. 
+
+If you want to learn more about creating good readme files then refer the following [guidelines](https://docs.microsoft.com/en-us/azure/devops/repos/git/create-a-readme?view=azure-devops). You can also seek inspiration from the below readme files:
+- [ASP.NET Core](https://github.com/aspnet/Home)
+- [Visual Studio Code](https://github.com/Microsoft/vscode)
+- [Chakra Core](https://github.com/Microsoft/ChakraCore)
\ No newline at end of file

azure-pipelines.yml

+name: $(Date:yyyyMMdd)$(Rev:.r)
+
+trigger:
+  - master
+  - feature/*
+
+variables:
+  - name: splitPipelineEnabled
+    value: false
+
+resources:
+  repositories:
+    - repository: templates
+      type: git
+      name: azure-pipeline-templates
+      ref: master
+    - repository: variables
+      type: git
+      name: vsts-release-management
+      ref: master
+  pipelines:
+    - pipeline: Automation-Tests
+      project: PDI
+      source: Automation-Tests
+      branch: master
+    - pipeline: vsts-testresult-uploader
+      project: PDI
+      source: vsts-testresult-uploader
+      branch: master
+    - pipeline: pds-restassured-tests
+      project: PDI
+      source: pds-restassured-tests
+      branch: master
+    - pipeline: perf-tests
+      project: PDI
+      source: perf-tests
+      branch: master
+
+stages:
+- template: build-stage.yml@templates
+  parameters:
+    name: osdu-writeback-publisher
+
+- ${{ if eq( variables['splitPipelineEnabled'], 'false' ) }}:
+  - template: run-deploy-stage-list.yml@templates
+    parameters:
+      name: osdu-writeback-publisher
+      restAssuredSmokeTest:
+        enabled: true
+        tagName: SmokeTestsWriteBackService
+      restAssuredSanityTest:
+        enabled: true
+        tagName: WitebackSanityTest
+      smokeTest:
+        enabled: false
+        testXMLFile: PDISmokeTest-WRITE-BACK-SERVICE.xml
+      sanityTest:
+        enabled: false
+        testXMLFile: PPS-SanityTest-WriteBackAPI.xml
+      regressionTest:
+        enabled: false
+
+- ${{ if eq( variables['splitPipelineEnabled'], 'true' ) }}:
+  - template: run-deploy-stage-list-sp.yml@templates
+    parameters:
+      name: osdu-writeback-publisher
+      restAssuredSmokeTest:
+        enabled: true
+        tagName: SmokeTestsWriteBackService
+      restAssuredSanityTest:
+        enabled: true
+        tagName: WitebackSanityTest  
+      smokeTest:
+        enabled: false
+        testXMLFile: PDISmokeTest-WRITE-BACK-SERVICE.xml
+      sanityTest:
+        enabled: false
+        testXMLFile: PPS-SanityTest-WriteBackAPI.xml
+      regressionTest:
+        enabled: false
\ No newline at end of file

build.sbt

+import org.opengroup.osdu.production.dependencies.CommanCompilerFlags._
+import org.opengroup.osdu.production.dependencies.CommonLibs._
+import sbt.Keys._
+import sbt._
+
+lazy val scalaVer = "2.12.4"
+lazy val osduOrg = "org.opengroup.osdu.production"
+lazy val ItTest = config("it") extend Test
+
+fork in IntegrationTest := true
+parallelExecution in IntegrationTest := false
+
+coverageExcludedPackages := "org.opengroup.osdu.production.domain.generated"
+coverageMinimum := 90
+coverageFailOnMinimum := true
+
+scalacOptions ++= org.opengroup.osdu.production.dependencies.CommanCompilerFlags.scalaOptions
+scalacOptions --= Seq("-Ywarn-dead-code")
+
+lazy val core =
+  Project("osdu-writeback-publisher", file("."))
+    .configs(IntegrationTest)
+    .configs(ItTest)
+    .settings(inConfig(ItTest)(Defaults.testSettings) : _*)
+    .settings(
+      organization := osduOrg,
+      scalaVersion := scalaVer,
+      javacOptions ++= javac18,
+      fork in Test := true,
+      resolvers ++= Seq(GCSResolver.forBucket("pdi-ivy-snapshots-repo"), GCSResolver.forBucket("pdi-ivy-release-repo")),
+      libraryDependencies ++= Seq(
+        typeSafeconfig,
+        scalactic,
+        scalaCheck,
+        scalaLogging,
+        scalaTest,
+        akkaActor,
+        akkaTestKit,
+        akkaSlf4j,
+        akkaStream,
+        json4sNative,
+        scoverage,
+        mockito,
+        akkaQuartzScheduler exclude("org.quartz-scheduler" , "quartz") exclude("com.mchange" , "c3p0"),
+        "org.quartz-scheduler" % "quartz" % "2.3.2",
+        "com.mchange" % "c3p0" % "0.9.5.4",
+        "com.google.protobuf" % "protobuf-java" % "3.12.0",
+        "org.opengroup.osdu.production" %% "osdu-scala-commons" % "0.3.36" exclude("org.apache.httpcomponents", "httpclient"),
+        "org.opengroup.osdu.production" %% "osdu-akka-stream-commons" % "0.2.26" exclude("org.apache.httpcomponents", "httpclient"),
+        "org.opengroup.osdu.production" %% "osdu-security-commons" % "0.1.40" exclude ("com.google.oauth-client", "google-oauth-client-1.25.0") exclude ("com.google.oauth-client", "google-oauth-client-1.27.0") exclude("org.apache.httpcomponents", "httpclient")
+      ).map(_.exclude("org.slf4j", "slf4j-log4j12"))
+       .map(_.exclude("com.google.guava","guava-20.0"))
+		   .map(_.exclude("com.google.oauth-client", "google-oauth-client"))
+      ).enablePlugins(JavaAppPackaging)
+       .enablePlugins(UniversalPlugin)
+
+name in Universal := name.value
+packageName in Universal := packageName.value
+mainClass in Compile := Some("org.opengroup.osdu.production.App")
+
+
+publishArtifact in(Compile, packageDoc) := false
+publishArtifact in(Universal, packageDoc) := false
+publishArtifact in(Universal, packageSrc) := false
+
+lazy val ClasspathPattern = "declare -r app_classpath=\"(.*)\"\n".r
+
+bashScriptDefines := bashScriptDefines.value.map {
+  case ClasspathPattern(classpath) => "declare -r app_classpath=\".:" + classpath + "\"\n"
+  case _@entry => entry
+}
+artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
+  artifact.name + "-" + module.revision + "-" + artifact.`type` + "." + artifact.extension
+}

build.sh

+#!/bin/bash
+
+
+declare -A envVars=( ["BUILD_BUILDNUMBER"]=$BUILD_BUILDNUMBER ["CONTAINER_REGISTRY_NAME"]=$CONTAINER_REGISTRY_NAME ["SONAR_URL"]=$SONAR_URL
+                     ["NEXUS_USERNAME"]=$NEXUS_USERNAME ["NEXUS_URL"]=$NEXUS_URL
+                     ["SONAR_LOGIN"]=$1 ["NEXUS_PASSWORD"]=$2 ["BINTRAY_PASSWORD"]=$3 )
+
+declare -a missingEnvVars
+
+for key in ${!envVars[@]}
+do
+    if [[ -z ${envVars[${key}]} ]]
+    then
+        missingEnvVars+=( ${key} )
+    fi
+done
+
+if [ ${#missingEnvVars[@]} -gt 0 ]
+then
+    joined=$(printf ", %s" "${missingEnvVars[@]}")
+    echo "Missing ${joined:1}"
+    exit 1
+fi
+
+export SONAR_LOGIN=$1
+export NEXUS_PASSWORD=$2
+export BINTRAY_PASSWORD=$3
+
+sed -i -e "s/PROJECT_BUILD_VERSION/${BUILD_BUILDNUMBER}/g" sonar-project.properties
+sed -i -e "s/GOOGLE_PROJECT_NAME/${GOOGLE_PROJECT_NAME}/g" ./src/main/resources/local.conf
+sed -i -e "s/GLOBAL_GOOGLE_PROJECT_NAME/${GLOBAL_GOOGLE_PROJECT_NAME}/g" src/it/resources/it-test.conf
+sed -i -e "s/GOOGLE_PROJECT_NAME/${GOOGLE_PROJECT_NAME}/g" src/it/resources/it-test.conf
+
+sbt -sbt-launch-repo https://repo1.maven.org/maven2 clean coverage test it:test coverageReport coverageOff
+#Fail if coverage has dropped
+rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi
+sbt universal:packageBin
+
+if [ ! -f /usr/bin/sonar-scanner ]
+then
+    wget https://dl.bintray.com/sonarsource/SonarQube/org/sonarsource/scanner/cli/sonar-scanner-cli/3.0.3.778/sonar-scanner-cli-3.0.3.778.zip && unzip sonar-scanner-cli-3.0.3.778.zip
+    cp -fR sonar-scanner-3.0.3.778/* /usr/
+fi
+sonar-scanner -Dsonar.host.url=${SONAR_URL} -Dsonar.login=${SONAR_LOGIN}
+tar -cvzf coverage.tar.gz target/scala-2.12/scoverage-report
+cp target/universal/osdu-writeback-publisher.zip build/osdu-writeback-publisher.zip
+unzip build/osdu-writeback-publisher -d build
+rm build/osdu-writeback-publisher.zip
+cd build
+docker build -t gcr.io/${CONTAINER_REGISTRY_NAME}/osdu-writeback-publisher:${BUILD_BUILDNUMBER} \
+  -t gcr.io/${CONTAINER_REGISTRY_NAME}/osdu-writeback-publisher:latest \
+  .

build/Dockerfile

+FROM openjdk:8u265-jre-slim
+COPY osdu-writeback-publisher /opt/osdu-writeback-publisher
+COPY run.sh /opt/osdu-writeback-publisher/bin
+RUN cd /opt \
+    && chmod +x -R  /opt/osdu-writeback-publisher/bin \
+    && apt-get update && apt-get install --only-upgrade bash && rm -rf /var/lib/apt/lists/*
+WORKDIR /opt/osdu-writeback-publisher/bin
+ENTRYPOINT [ "/opt/osdu-writeback-publisher/bin/run.sh", "app-conf" ]
\ No newline at end of file

build/run.sh

+#!/bin/sh
+ln -s /opt/config-mount/app-config.conf /opt/osdu-writeback-publisher/bin/app-conf.conf
+
+/opt/osdu-writeback-publisher/bin/osdu-writeback-publisher $1

helm-chart/writeback-publisher/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm-chart/writeback-publisher/Chart.yaml

+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: osdu-writeback-publisher
+version: 0.1.0

helm-chart/writeback-publisher/files/env.conf

+cloud {
+  global-project {
+    name = "{{ .Values.globalResourcesProjectId }}"
+  }
+  pubsub {
+    writeback-data-publish-gateway{
+      subscription {
+        name = "writeback-data-publish-gateway-subscriber-{{ .Values.pds.envTag }}"
+        parallel-pull-count = 10
+        ack-deadline-seconds = 60
+        retain-acked-messages = true
+      }
+      topic {
+        name = "writeback-data-publish-gateway-{{ .Values.pds.envTag }}"
+      }
+    }
+
+    writeback-data-gateway{
+      subscription {
+        name = "writeback-data-gateway-subscriber-{{ .Values.pds.envTag }}"
+        parallel-pull-count = 10
+        ack-deadline-seconds = 60
+        retain-acked-messages = true
+      }
+      topic {
+        name = "writeback-data-gateway-{{ .Values.pds.envTag }}"
+      }
+    }
+
+    timeseries-data-gateway {
+      topic {
+        name = "{{ .Values.pds.envTag }}-data-gateway"
+      }
+    }
+    max-message-size =  {{ .Values.pds.pubSubMaxMessageSizeInBytes }} // 8 MB
+  }
+
+}
+
+batch {
+  maxRetries = 10
+  delay = 2
+  batchSize = 100
+  maxParallel=10
+}
+
+dependent-services{
+  source-reference-service {
+    hostname = "source-reference-service",
+    port = 9090,
+    secret = "",
+    uri {
+      createStream = "/reference/api/v1/streams"
+    }
+  }
+
+  pdm-service {
+    hostname = "pdm-service",
+    port = 9090,
+    secret = "",
+    uri {
+      getPropertyByPropDesc = "/model/api/v3/properties/PROP_DESC"
+    }
+  }
+
+  structure-query-service {
+    hostname = "structure-query-service",
+    port = 9090,
+    secret = "",
+    uri {
+      getXrefFromUUID = "/structure/api/v1/getXrefsFromUUID/%s",
+      getElementFromXref = "/structure/api/v1/elements/getElementByXref?xref=%s"
+    }
+  }
+
+  structure-storage-service {
+    hostname="structure-service",
+    port=9090,
+    secret = "",
+    uri {
+      linkStream ="/metadata/api/v1/elements/%s/measurements"
+    }
+  }
+}
+
+http{
+  queue{
+    size=20
+  }
+}
+
+quartz {
+  schedules {
+    probeSchedule {
+      description = "A cron job that runs every minute",
+      expression = "0 * * ? * * *"
+    }
+  }
+}

helm-chart/writeback-publisher/templates/_helpers.tpl

+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "osdu-writeback-publisher.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "osdu-writeback-publisher.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "osdu-writeback-publisher.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "osdu-writeback-publisher.labels" -}}
+app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
+helm.sh/chart: {{ include "osdu-writeback-publisher.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}

helm-chart/writeback-publisher/templates/configmap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "osdu-writeback-publisher.fullname" . }}-config
+data:
+  app-config.conf: |
+{{ tpl (.Files.Get "files/env.conf") . | indent 4 }}
\ No newline at end of file

helm-chart/writeback-publisher/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "osdu-writeback-publisher.fullname" . }}
+  labels:
+{{ include "osdu-writeback-publisher.labels" . | indent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+      labels:
+        app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.pds.imageRepository }}/{{ .Chart.Name }}:{{ .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          volumeMounts:
+            - name: application-config
+              mountPath: /opt/config-mount
+            - name: secret-config
+              mountPath: /opt/secret-mount
+          livenessProbe:
+            exec:
+              command:
+                - rm
+                - /opt/osdu-writeback-publisher/bin/alive
+            initialDelaySeconds: 90
+            periodSeconds: 120
+            failureThreshold: 3
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: /etc/ssl/certs
+        - name: application-config
+          configMap:
+            name: {{ template "osdu-writeback-publisher.fullname" . }}-config
+        - name: secret-config
+          secret:
+            secretName: {{ template "osdu-writeback-publisher.fullname" . }}-secret
\ No newline at end of file

helm-chart/writeback-publisher/templates/secret.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "osdu-writeback-publisher.fullname" . }}-secret
+type: Opaque

helm-chart/writeback-publisher/templates/service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "osdu-writeback-publisher.fullname" . }}
+  labels:
+{{ include "osdu-writeback-publisher.labels" . | indent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: {{ include "osdu-writeback-publisher.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
\ No newline at end of file

helm-chart/writeback-publisher/values.yaml

+replicaCount: 1
+
+image:
+  pullPolicy: IfNotPresent
+
+service:
+  type: ClusterIP
+  port: 9090
+  
+resources:
+  requests:
+    memory: 512Mi
+    cpu: 10m
+  limits:
+    memory: 2Gi
+    cpu: 100m
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
\ No newline at end of file

project/build.properties

+sbt.version=1.2.8
\ No newline at end of file

project/plugins.sbt

+
+//format: OFF
+
+resolvers += Classpaths.typesafeReleases
+resolvers += Resolver.url("bintray-sbt-plugin-releases",url("https://dl.bintray.com/content/sbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
+resolvers += GCSResolver.forBucket("pdi-ivy-release-repo")
+val typesafeRepo = "com.typesafe.sbt"
+
+addSbtPlugin(typesafeRepo             % "sbt-git"                                     % "0.9.3")
+addSbtPlugin(typesafeRepo             % "sbt-license-report"                          % "1.2.0")
+addSbtPlugin(typesafeRepo             % "sbt-native-packager"                         % "1.3.2")
+addSbtPlugin("org.scoverage"          % "sbt-scoverage"                               % "1.5.1")
+addSbtPlugin("org.scalariform"        % "sbt-scalariform"                             % "1.8.0")
+addSbtPlugin("org.scalastyle"         %% "scalastyle-sbt-plugin"                      % "1.0.0")
+addSbtPlugin("com.github.gseitz"      % "sbt-release"                                 % "1.0.7")
+addSbtPlugin("net.virtual-void"       % "sbt-dependency-graph"                        % "0.9.2")
+addSbtPlugin("net.vonbuchholtz"       % "sbt-dependency-check"                        % "2.0.0")
+addSbtPlugin("com.thesamet"           % "sbt-protoc"                                  % "0.99.12")
+addSbtPlugin("com.lightbend"          % "sbt-google-cloud-storage"                    % "0.0.10")
+addSbtPlugin("org.opengroup.osdu.production"          % "osdu-common-deps"            % "0.0.57")

project/project/plugins.sbt

+import sbt.{Resolver, addSbtPlugin}
+//format: OFF
+
+addSbtPlugin("com.lightbend"          % "sbt-google-cloud-storage"                    % "0.0.10")
\ No newline at end of file

pull_request_template.md

+## Description
+
+Add your Code change description here.
+
+## Type of changes is acceptable ( New Feature, Fix, Breaking Change )
+- [ ] New Feature
+- [ ] Fix
+- [ ] Breaking Change
+
+## Unit Test added in the build process and results are..
+
+- [ ] Success 
+- [ ] Failed
+- [ ] Not Applicable
+
+## Integration Test added in the build process and results are..
+
+- [ ] Success 
+- [ ] Failed
+- [ ] Not Applicable