merge from OSDU-delfi

replace RedisCache with VmCache (loadingCache)

# Conflicts:
#	provider/partition-azure/src/main/resources/application.properties

provider/partition-azure/devops/stages.yml

@@ -89,16 +89,6 @@ stages:
           zip -r testing.zip testing
           cp testing.zip $(build.artifactstagingdirectory)
         displayName: 'Zip integration tests and upload it as artifact'
-      - bash: |
-          #!/bin/bash
-          set -e
-          # Upload redis deployment scripts as artifact
-          pushd provider/partition-azure/redis-deployment
-          cp master-deployment.yaml $(build.artifactstagingdirectory)
-          cp redis-service.yaml $(build.artifactstagingdirectory)
-          cp sentinel-deployment.yaml $(build.artifactstagingdirectory)
-          cp slave-deployment.yaml $(build.artifactstagingdirectory)
-          popd
       - upload: $(Build.ArtifactStagingDirectory)
         artifact: drop_${{environment.name}}
 
@@ -152,19 +142,6 @@ stages:
                 set -e
                 kubectl --kubeconfig $(kubeConfig.secureFilePath) rollout restart deployment/partition-aks
               displayName: 'kubectl rollout restart deployment/partition-aks'
-            - bash: |
-                #!/bin/bash
-                set -e
-                pushd $(Pipeline.Workspace)/drop_${{environment.name}}
-                kubectl --kubeconfig $(kubeConfig.secureFilePath) apply -f redis-service.yaml
-                sleep 5
-                kubectl --kubeconfig $(kubeConfig.secureFilePath) apply -f master-deployment.yaml
-                sleep 5
-                kubectl --kubeconfig $(kubeConfig.secureFilePath) apply -f sentinel-deployment.yaml
-                sleep 5
-                kubectl --kubeconfig $(kubeConfig.secureFilePath) apply -f slave-deployment.yaml
-                popd
-              displayName: 'deploy redis cluster'
             - ${{ if or(eq(environment.name, 'devint'), eq(environment.name, 'qa')) }}:
               - bash: |
                   #!/bin/bash

provider/partition-azure/redis-deployment/master-deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis-partition-master
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis-partition-master
-  template:
-    metadata:
-      name: redis-partition-master
-      labels:
-        app: redis-partition-master
-        master: "true"
-    spec:
-      volumes:
-      - hostPath:
-          path: /tmp/data/1
-        name: redis-directory-binding
-
-      restartPolicy: Always
-
-      containers:
-        - name: redis-partition-master
-          image: delfi.azurecr.io/redis-partition-cluster:latest
-
-          resources:
-            requests:
-              memory: "100Mi"
-              cpu: .2
-            limits:
-              memory: "200Mi"
-              cpu: .5
-
-          imagePullPolicy: Always
-
-          ports:
-          - containerPort: 6379
-
-          volumeMounts:
-          - mountPath: /redis-data
-            name: redis-directory-binding
-            readOnly: false
-
-          env:
-          - name: MASTER
-            value: "true"
-      imagePullSecrets:
-        - name: acr
\ No newline at end of file

provider/partition-azure/redis-deployment/redis-docker/Dockerfile

-FROM redis:latest
-
-MAINTAINER Alok Joshi
-
-RUN apt-get install bash \
-    sed
-
-RUN mkdir /redis-master && \
-    mkdir /redis-slave
-
-COPY redis-master.conf /redis-master/redis.conf
-COPY redis-slave.conf /redis-slave/redis.conf
-COPY entrypoint.sh /entrypoint.sh
-
-RUN chmod 777 /entrypoint.sh
-
-CMD [ "/entrypoint.sh" ]
-
-ENTRYPOINT [ "bash", "-c" ]
\ No newline at end of file

provider/partition-azure/redis-deployment/redis-docker/build-and-push.sh

-#!/bin/bash
-
-set -e
-
-export imageName="redis-partition-cluster"
-export tag="v1"
-
-docker login -u ${dockerId} -p ${dockerPassword} ${dockerId}.azurecr.io
-docker build -t ${dockerId}.azurecr.io/${imageName}:${tag} .
-echo 'Image built'
-
-docker push ${dockerId}.azurecr.io/${imageName}
-docker tag ${dockerId}.azurecr.io/${imageName}:${tag} ${dockerId}.azurecr.io/${imageName}:latest
-
-echo 'Added ${dockerId}.azurecr.io/${imageName}:latest tag to ${dockerId}.azurecr.io/${imageName}:${tag}'
-docker push ${dockerId}.azurecr.io/${imageName}:${tag}
-
-echo 'Pushing ${dockerId}.azurecr.io/${imageName}:${tag}'
-docker push ${dockerId}.azurecr.io/${imageName}:latest
-echo 'Pushed ${dockerId}.azurecr.io/${imageName}:latest'
\ No newline at end of file

provider/partition-azure/redis-deployment/redis-docker/entrypoint.sh

-#!/bin/bash
-
-#  @Description:
-#    Entrypoint script for deploying redis HA via Sentinel in a kubernetes cluster
-#    This script expects following environment variables to be set,
-#    1. SENTINEL: true if this is sentinel instance, else false.
-#    2. MASTER: true if this is master instance, this is helpful when starting the cluster for the first time.
-#    3. REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_HOST: this is service name of sentinel, check the yaml.
-#    4. REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_PORT: this is service port of sentinel.
-#    5. REDIS_HA_CLUSTER_STARTUP_REDIS_MASTER_SERVICE_SERVICE_HOST: this is master's service name, this is needed when sentinel starts for the first time.
-#    6. REDIS_HA_CLUSTER_STARTUP_REDIS_MASTER_SERVICE_SERVICE_PORT: this is master's port, is needed when sentinel starts for the first time.
-
-#  This method launches redis instance which assumes itself as master
-function launchmaster() {
-  echo "Starting Redis instance as Master.."
-
-  echo "while true; do   sleep 2;   export master=\$(hostname -i);   echo \"Master IP is Me : \${master}\";   echo \"Setting STARTUP_MASTER_IP in redis\";   redis-cli -a ${REDIS_DEFAULT_PASSWORD} -h \${master} set STARTUP_MASTER_IP \${master};   if [ \$? == \"0\" ]; then     echo \"Successfully set STARTUP_MASTER_IP\";     break;   fi;   echo \"Connecting to master \${master} failed.  Waiting...\";   sleep 5; done" > insert_master_ip.sh
-
-  bash insert_master_ip.sh &
-
-  sed -i "s/REDIS_DEFAULT_PASSWORD/${REDIS_DEFAULT_PASSWORD}/" /redis-master/redis.conf
-  redis-server /redis-master/redis.conf --protected-mode no
-
-}
-
-#  This method launches sentinels
-function launchsentinel() {
-  echo "Starting Sentinel.."
-  sleep_for_rand_int=$(awk -v min=2 -v max=7 'BEGIN{srand(); print int(min+rand()*(max-min+1))}')
-  sleep ${sleep_for_rand_int}
-
-  while true; do
-    echo "Trying to connect to Sentinel Service"
-    master=$(redis-cli -h ${REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_HOST} -p ${REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_PORT} --csv SENTINEL get-master-addr-by-name mymaster | tr ',' ' ' | cut -d' ' -f1)
-    if [[ -n ${master} ]]; then
-      echo "Connected to Sentinel Service and retrieved Redis Master IP as ${master}"
-      master="${master//\"}"
-    else
-      echo "Unable to connect to Sentinel Service, probably because I am first Sentinel to start. I will try to find STARTUP_MASTER_IP from the redis service"
-      master=$(redis-cli -a ${REDIS_DEFAULT_PASSWORD} -h ${REDIS_HA_CLUSTER_STARTUP_REDIS_MASTER_SERVICE_SERVICE_HOST} -p ${REDIS_HA_CLUSTER_STARTUP_REDIS_MASTER_SERVICE_SERVICE_PORT} get STARTUP_MASTER_IP)
-      if [[ -n ${master} ]]; then
-        echo "Retrieved Redis Master IP as ${master}"
-      else
-        echo "Unable to retrieve Master IP from the redis service. Waiting..."
-        sleep 10
-        continue
-      fi
-    fi
-
-    redis-cli -a ${REDIS_DEFAULT_PASSWORD} -h ${master} INFO
-    if [[ "$?" == "0" ]]; then
-      break
-    fi
-    echo "Connecting to master failed.  Waiting..."
-    sleep 10
-  done
-
-  sentinel_conf=sentinel.conf
-
-  echo "sentinel monitor mymaster ${master} 6379 2" > ${sentinel_conf}
-  echo "sentinel down-after-milliseconds mymaster 5000" >> ${sentinel_conf}
-  echo "sentinel failover-timeout mymaster 60000" >> ${sentinel_conf}
-  echo "sentinel parallel-syncs mymaster 1" >> ${sentinel_conf}
-  echo "bind 0.0.0.0" >> ${sentinel_conf}
-  echo "sentinel auth-pass mymaster ${REDIS_DEFAULT_PASSWORD}" >> ${sentinel_conf}
-
-  redis-sentinel ${sentinel_conf} --protected-mode no
-}
-
-#  This method launches slave instances
-function launchslave() {
-  echo "Starting Redis instance as Slave , Master IP $1"
-
-  while true; do
-    echo "Trying to retrieve the Master IP again, in case of failover master ip would have changed."
-    master=$(redis-cli -h ${REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_HOST} -p ${REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_PORT} --csv SENTINEL get-master-addr-by-name mymaster | tr ',' ' ' | cut -d' ' -f1)
-    if [[ -n ${master} ]]; then
-      master="${master//\"}"
-    else
-      echo "Failed to find master."
-      sleep 60
-      continue
-    fi
-    redis-cli -a ${REDIS_DEFAULT_PASSWORD} -h ${master} INFO
-    if [[ "$?" == "0" ]]; then
-      break
-    fi
-    echo "Connecting to master failed.  Waiting..."
-    sleep 10
-  done
-
-  sed -i "s/%master-ip%/${master}/" /redis-slave/redis.conf
-  sed -i "s/%master-port%/6379/" /redis-slave/redis.conf
-  sed -i "s/REDIS_DEFAULT_PASSWORD/${REDIS_DEFAULT_PASSWORD}/" /redis-slave/redis.conf
-  redis-server /redis-slave/redis.conf --protected-mode no
-}
-
-
-#  This method launches either slave or master based on some parameters
-function launchredis() {
-  echo "Launching Redis instance"
-
-  # Loop till I am able to launch slave or master
-  while true; do
-    # I will check if sentinel is up or not by connecting to it.
-    echo "Trying to connect to sentinel, to retireve master's ip"
-    master=$(redis-cli -h ${REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_HOST} -p ${REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_PORT} --csv SENTINEL get-master-addr-by-name mymaster | tr ',' ' ' | cut -d' ' -f1)
-
-    # Is this instance marked as MASTER, it will matter only when the cluster is starting up for first time.
-    if [[ "${MASTER}" == "true" ]]; then
-      echo "MASTER is set to true"
-      # If I am able get master ip, then i will connect to the master, else i will asume the role of master
-      if [[ -n ${master} ]]; then
-        echo "Connected to Sentinel, this means it is not first time start, hence will start as a slave"
-        launchslave ${master}
-        exit 0
-      else
-        launchmaster
-        exit 0
-      fi
-    fi
-
-    # If I am not master, then i am definitely slave.
-    if [[ -n ${master} ]]; then
-      echo "Connected to Sentinel and Retrieved Master IP ${master}"
-      launchslave ${master}
-      exit 0
-    else
-      echo "Connecting to sentinel failed, Waiting..."
-      sleep 10
-    fi
-  done
-}
-
-export REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_HOST="redis-partition-sentinel-service"
-export REDIS_HA_CLUSTER_SENTINEL_SERVICE_SERVICE_PORT="26379"
-export REDIS_HA_CLUSTER_STARTUP_REDIS_MASTER_SERVICE_SERVICE_HOST="redis-partition-master-service"
-export REDIS_HA_CLUSTER_STARTUP_REDIS_MASTER_SERVICE_SERVICE_PORT="6379"
-
-# TODO: should not be hardcoded
-export REDIS_DEFAULT_PASSWORD="admin"
-
-if [[ "${SENTINEL}" == "true" ]]; then
-  launchsentinel
-  exit 0
-fi
-
-launchredis
\ No newline at end of file

provider/partition-azure/redis-deployment/redis-service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    name: redis-partition-sentinel-service
-  name: redis-partition-sentinel-service
-spec:
-  ports:
-  - port: 26379
-  selector:
-    app: redis-partition-sentinel
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    name: redis-partition-master-service
-  name: redis-partition-master-service
-spec:
-  ports:
-  - port: 6379
-  selector:
-    app: redis-partition-master
\ No newline at end of file

provider/partition-azure/redis-deployment/sentinel-deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis-partition-sentinel
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: redis-partition-sentinel
-  template:
-    metadata:
-      name: redis-partition-sentinel
-      labels:
-        app: redis-partition-sentinel
-    spec:
-      restartPolicy: Always
-
-      containers:
-        - name: redis-partition-sentinel
-          image: delfi.azurecr.io/redis-partition-cluster:latest
-
-          resources:
-            requests:
-              memory: "100Mi"
-              cpu: .2
-            limits:
-              memory: "200Mi"
-              cpu: .5
-
-          imagePullPolicy: Always
-
-          ports:
-          - containerPort: 26379
-
-          env:
-          - name: SENTINEL
-            value: "true"
-      imagePullSecrets:
-       - name: acr
\ No newline at end of file

provider/partition-azure/redis-deployment/slave-deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis-partition-slave
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis-partition-slave
-  template:
-    metadata:
-      name: redis-partition-slave
-      labels:
-        app: redis-partition-slave
-        slave: "true"
-    spec:
-      volumes:
-      - hostPath:
-          path: /tmp/data/2
-        name: redis-directory-binding
-
-      restartPolicy: Always
-
-      containers:
-        - name: redis-partition-slave
-          image: delfi.azurecr.io/redis-partition-cluster:latest
-
-          resources:
-            requests:
-              memory: "100Mi"
-              cpu: .2
-            limits:
-              memory: "200Mi"
-              cpu: .5
-
-          imagePullPolicy: Always
-
-          ports:
-          - containerPort: 6379
-
-          volumeMounts:
-          - mountPath: /redis-data
-            name: redis-directory-binding
-            readOnly: false
-      imagePullSecrets:
-        - name: acr
\ No newline at end of file

provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceCacheImpl.java

 package org.opengroup.osdu.partition.provider.azure.service;
 
-import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.cache.VmCache;
 import org.opengroup.osdu.partition.model.PartitionInfo;
 import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 @Service
-public class PartitionServiceCacheImpl extends RedisCache<String, PartitionInfo> implements IPartitionServiceCache {
-    public PartitionServiceCacheImpl(@Value("${REDIS_HOST}") final String host
-            , @Value("${REDIS_PORT}") final int port) {
-        super(host, port, 60*60, String.class, PartitionInfo.class);
+public class PartitionServiceCacheImpl extends VmCache<String, PartitionInfo> implements IPartitionServiceCache {
+    public PartitionServiceCacheImpl() {
+        super(5*60, 1000);
     }
 }

provider/partition-azure/src/main/resources/application.properties

@@ -3,7 +3,6 @@ server.servlet.contextPath=/api/partition/v1
 logging.level.org.springframework.web=DEBUG
 JAVA_OPTS=-Dserver.port=80
 server.port=8080
-spring.data.redis.repositories.enabled=false
 springfox.documentation.swagger.v2.path=/api-docs
 
 #logging configuration
@@ -24,8 +23,4 @@ azure.application-insights.instrumentation-key=${appinsights_key}
 # Azure service connection properties
 AZURE_CLIENT_ID=${AZURE_CLIENT_ID}
 AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}
-AZURE_TENANT_ID=${AZURE_TENANT_ID}
-
-# Redis cluster properties
-REDIS_HOST=${REDIS_PARTITION_HOST:127.0.0.1}
-REDIS_PORT=${REDIS_PARTITION_PORT:6379}
\ No newline at end of file
+AZURE_TENANT_ID=${AZURE_TENANT_ID}
\ No newline at end of file