Merge remote-tracking branch 'origin/master' into dev

.gitlab-ci.yml

@@ -17,6 +17,7 @@ variables:
   OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-legal --set data.limits_cpu=1.5 --set data.limits_memory=512M"
   OSDU_GCP_HELM_CONFIG_SERVICE: legal-config
   OSDU_GCP_HELM_DEPLOYMENT_SERVICE: legal-deploy
+  OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
 
   IBM_BUILD_SUBDIR: provider/legal-ibm
   IBM_INT_TEST_SUBDIR: testing/legal-test-ibm

devops/azure/chart/helm-config.yaml

@@ -3,6 +3,8 @@ global:
 
   # Service(s) Replica Count
   replicaCount: 2
+  nodepool: services
+  isAutoscalingEnabled: false
 
 ################################################################################
 # Specify the Gitlab branch being used for image creation

devops/azure/chart/templates/deployment.yaml

@@ -28,6 +28,10 @@ spec:
         app: {{ .Chart.Name }}
         aadpodidbinding: osdu-identity
     spec:
+{{- if .Values.global.isAutoscalingEnabled }}
+      nodeSelector:
+        nodepool: {{ .Values.global.nodepool }} 
+{{- end }}
       volumes:
       - name: azure-keyvault
         csi:
@@ -48,8 +52,14 @@ spec:
             cpu: "300m"
         readinessProbe:
           httpGet:
-            path: /api/legal/v1/swagger-ui.html
-            port: 80
+            path: /actuator/health
+            port: 8081
+        livenessProbe:
+          httpGet:
+            path: /actuator/health
+            port: 8081
+          initialDelaySeconds: 250
+          periodSeconds: 10
         volumeMounts:
         - name: azure-keyvault
           mountPath: "/mnt/azure-keyvault"
@@ -118,5 +128,7 @@ spec:
             value: http://partition/api/partition/v1
           - name: azure_istioauth_enabled
             value: "true"
+          - name: SPRING_CONFIG_NAME
+            value: "common,application"
           - name: azure_activedirectory_AppIdUri
             value: "api://$(aad_client_id)"

devops/azure/chart/values.yaml

@@ -14,6 +14,8 @@
 
 global:
   replicaCount: 1
+  nodepool: services
+  isAutoscalingEnabled: false
 
 image:
   repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal

devops/gcp/deploy/values.yaml

@@ -4,9 +4,9 @@
 
 data:
   requests_cpu: "0.25"
-  requests_memory: "128M"
-  limits_cpu: "0.5"
-  limits_memory: "256M"
+  requests_memory: "256M"
+  limits_cpu: "1.5"
+  limits_memory: "1G"
   serviceAccountName: ""
   image: ""
 

docs/api/compliance_openapi.yaml

@@ -19,6 +19,8 @@ security:
 tags:
   - name: LegalTag
     description: ''
+  - name: info
+    description: "Version info endpoint"
 schemes:
   - https
 consumes:
@@ -308,6 +310,21 @@ paths:
         - Bearer: []
         - google_id_token: []
         - sauth_id_token: []
+  /info:
+    get:
+      tags:
+        - info
+      summary: "Version info"
+      description: "For deployment available public `/info` endpoint, \
+        \ which provides build and git related information."
+      operationId: "Version info"
+      produces:
+        - "application/json"
+      responses:
+        200:
+          description: "Version info."
+          schema:
+            $ref: "#/definitions/VersionInfo"
 securityDefinitions:
   Bearer:
     type: apiKey
@@ -483,3 +500,43 @@ definitions:
         example: '2025-12-25'
         description: The optional expiration date of the contract in the format YYYY-MM-DD
     description: The model to update an existing LegalTag
+  VersionInfo:
+    type: "object"
+    properties:
+      groupId:
+        type: "string"
+        description: "Maven artifact group ID."
+      actifactId:
+        type: "string"
+        description: "Maven artifact ID."
+      version:
+        type: "string"
+        description: "Maven artifact version"
+      buildTime:
+        type: "string"
+        description: "Maven artifact build time"
+      branch:
+        type: "string"
+        description: "Current git branch"
+      commitId:
+        type: "string"
+        description: "Latest commit hash"
+      commitMessage:
+        type: "string"
+        description: "Latest commit message"
+      connectedOuterServices:
+        type: "array"
+        description: "Connected outer services information"
+        items:
+          $ref: "#/definitions/ConnectedOuterService"
+    description: "Version info."
+  ConnectedOuterService:
+    type: "object"
+    properties:
+      name:
+        type: "string"
+        description: "Connected outer service name."
+      version:
+        type: "string"
+        description: "Connected outer service version."
+    description: "Connected outer service information."
\ No newline at end of file

docs/tutorial/ComplianceService.md

@@ -15,6 +15,7 @@
 * [Compliance on consumption](#Compliance-on-consumption)
 * [The LegalTag Changed notification](#The-LegalTag-Changed-notification)
 * [Permissions](#Permissions)
+* [Version info endpoint](#version-info-endpoint)
 
 ## Introduction<a name="Introduction"></a>
 
@@ -509,3 +510,39 @@ If it has become incompliant, you must make sure associated data is no longer al
 If it is marked compliant, data that was not allowed for consumption can now be consumed through your services.
 
 [Back to table of contents](#TOC)
+
+## Version info endpoint
+For deployment available public `/info` endpoint, which provides build and git related information.
+#### Example response:
+```json
+{
+    "groupId": "org.opengroup.osdu",
+    "artifactId": "storage-gcp",
+    "version": "0.10.0-SNAPSHOT",
+    "buildTime": "2021-07-09T14:29:51.584Z",
+    "branch": "feature/GONRG-2681_Build_info",
+    "commitId": "7777",
+    "commitMessage": "Added copyright to version info properties file",
+    "connectedOuterServices": [
+      {
+        "name": "elasticSearch",
+        "version":"..."
+      },
+      {
+        "name": "postgresSql",
+        "version":"..."
+      },
+      {
+        "name": "redis",
+        "version":"..."
+      }
+    ]
+}
+```
+This endpoint takes information from files, generated by `spring-boot-maven-plugin`,
+`git-commit-id-plugin` plugins. Need to specify paths for generated files to matching
+properties:
+- `version.info.buildPropertiesPath`
+- `version.info.gitPropertiesPath`
+
+[Back to table of contents](#TOC)
\ No newline at end of file

legal-core/pom.xml

@@ -181,7 +181,6 @@
 	</dependencies>
 
 	<build>
-		<defaultGoal>test</defaultGoal>
 		<plugins>
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
@@ -191,6 +190,9 @@
 					<useSystemClassLoader>false</useSystemClassLoader>
 					<threadCount>1</threadCount>
 				</configuration>
+				<goals>
+					<goal>test</goal>
+				</goals>
 			</plugin>
 		</plugins>
 	</build>

legal-core/src/main/java/org/opengroup/osdu/legal/api/InfoApi.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.legal.api;
+
+import java.io.IOException;
+import org.opengroup.osdu.core.common.info.VersionInfoBuilder;
+import org.opengroup.osdu.core.common.model.info.VersionInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping
+public class InfoApi {
+
+  @Autowired
+  private VersionInfoBuilder versionInfoBuilder;
+
+  @GetMapping(value = "/info", produces = MediaType.APPLICATION_JSON_VALUE)
+  public VersionInfo info() throws IOException {
+    return versionInfoBuilder.buildVersionInfo();
+  }
+}
\ No newline at end of file

legal-core/src/main/java/org/opengroup/osdu/legal/middleware/LegalFilter.java

@@ -2,9 +2,8 @@ package org.opengroup.osdu.legal.middleware;
 
 import java.io.IOException;
 import java.time.Duration;
-import java.util.List;
 import java.util.Map;
-
+import javax.inject.Inject;
 import javax.servlet.Filter;
 import javax.servlet.FilterChain;
 import javax.servlet.ServletException;
@@ -12,19 +11,16 @@ import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import javax.inject.Inject;
-
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService;
 import org.opengroup.osdu.core.common.http.ResponseHeadersFactory;
-import org.opengroup.osdu.core.common.model.http.Request;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.springframework.context.annotation.Lazy;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.Request;
+import org.opengroup.osdu.core.common.model.http.RequestInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Component;
 
-import org.opengroup.osdu.core.common.model.http.RequestInfo;
-
 @Component
 @Lazy
 public class LegalFilter implements Filter {
@@ -68,16 +64,21 @@ public class LegalFilter implements Filter {
         }finally {
             logRequest(httpServletRequest, httpServletResponse, startTime);
         }
-        
+
 	}
 
 	@Override
 	public void destroy() {
     }
-    
+
     private boolean validateIsHttps( HttpServletResponse httpServletResponse, HttpServletRequest httpServletRequest) {
         String uri = requestInfo.getUri();
-        if(!isLocalHost(uri) && !isCronJob(uri) && !isSwagger(uri) && !isHealthCheck(uri) && !isOptionsMethod(httpServletRequest)) {
+        if (!isLocalHost(uri)
+            && !isCronJob(uri)
+            && !isSwagger(uri)
+            && !isVersionInfo(uri)
+            && !isHealthCheck(uri)
+            && !isOptionsMethod(httpServletRequest)) {
             if(!hasJwt()) {
                 httpServletResponse.setStatus(401);
                 return false;
@@ -108,6 +109,11 @@ public class LegalFilter implements Filter {
     private boolean isCronJob(String uri) {
         return uri.contains("/jobs/updateLegalTagStatus");
     }
+
+    private boolean isVersionInfo(String uri) {
+        return uri.contains("/info");
+    }
+
     private boolean isSwagger(String uri) {
         return uri.contains("/swagger") || uri.contains("/v2/api-docs") || uri.contains("/configuration/ui") || uri.contains("/webjars/");
     }

legal-core/src/main/java/org/opengroup/osdu/legal/util/CloudConnectedOuterServicesBuilder.java

+package org.opengroup.osdu.legal.util;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.info.ConnectedOuterServicesBuilder;
+import org.opengroup.osdu.core.common.model.info.ConnectedOuterService;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConditionalOnMissingBean(type = "ConnectedOuterServicesBuilder")
+public class CloudConnectedOuterServicesBuilder implements ConnectedOuterServicesBuilder {
+
+  private static final String REDIS_PREFIX = "Redis-";
+
+  private List<RedisCache> redisCaches;
+
+  public CloudConnectedOuterServicesBuilder(List<RedisCache> redisCaches) {
+    this.redisCaches = redisCaches;
+  }
+
+  @Override
+  public List<ConnectedOuterService> buildConnectedOuterServices() {
+    return redisCaches.stream().map(this::fetchRedisInfo).collect(Collectors.toList());
+  }
+
+  private ConnectedOuterService fetchRedisInfo(RedisCache cache) {
+    String redisVersion = StringUtils.substringBetween(cache.info(), ":", "\r");
+    return ConnectedOuterService.builder()
+        .name(REDIS_PREFIX + StringUtils.substringAfterLast(cache.getClass().getName(), "."))
+        .version(redisVersion)
+        .build();
+  }
+}

pom.xml

@@ -8,7 +8,7 @@
         <maven.compiler.source>1.8</maven.compiler.source>
         <docker.image.prefix>opendes</docker.image.prefix>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <os-core-common.version>0.10.0</os-core-common.version>
+        <os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
         <snakeyaml.version>1.26</snakeyaml.version>
     </properties>
 
@@ -111,4 +111,46 @@
         </profile>
     </profiles>
 
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot-maven-plugin</artifactId>
+                    <executions>
+                        <execution>
+                            <id>build-info</id>
+                            <goals>
+                                <goal>build-info</goal>
+                            </goals>
+                        </execution>
+                    </executions>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+
+        <plugins>
+            <plugin>
+                <groupId>pl.project13.maven</groupId>
+                <artifactId>git-commit-id-plugin</artifactId>
+                <version>4.0.5</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <verbose>true</verbose>
+                    <dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
+                    <generateGitPropertiesFile>true</generateGitPropertiesFile>
+                    <generateGitPropertiesFilename>
+                        ${project.build.outputDirectory}/git.properties
+                    </generateGitPropertiesFilename>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
 </project>

provider/legal-aws/src/main/resources/application.properties

@@ -38,4 +38,4 @@ server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
 server.ssl.key-password=${SSL_KEY_PASSWORD:}
 server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
 
-spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
+spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
\ No newline at end of file

provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AADSecurityConfig.java

@@ -46,6 +46,7 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
                     "/swagger-resources/**",
                     "/configuration/security",
                     "/swagger",
+                    "/info",
                     "/swagger-ui.html",
                     "/webjars/**").permitAll()
             .anyRequest().authenticated()

provider/legal-azure/src/main/resources/application.properties

@@ -70,4 +70,5 @@ tenantFactoryImpl.required=true
 tenantInfo.container.name=TenantInfo
 
 # Disable keyVault for actuator health check
-management.health.azure-key-vault.enabled=false
\ No newline at end of file
+management.health.azure-key-vault.enabled=false
+management.server.port=8081

provider/legal-byoc/src/main/resources/application.properties

@@ -9,6 +9,4 @@ spring.security.user.roles=service.legal.admin
 REGION=us-central
 AUTHORIZE_API=notused
 LEGAL_HOSTNAME=notused
-CRON_JOB_IP=10.0.0.1
-
-
+CRON_JOB_IP=10.0.0.1
\ No newline at end of file

provider/legal-gcp/README.md

@@ -24,6 +24,26 @@ These instructions will get you a copy of the project up and running on your loc
 
 ### Run Locally
 
+### Environment Variables
+
+In order to run the service locally, you will need to have the following environment variables defined.
+
+| name | value | description | sensitive? | source |
+| ---  | ---   | ---         | ---        | ---    |
+| `LOG_PREFIX` | `legal` | Logging prefix | no | - |
+| `AUTHORIZE_API` | `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
+| `LEGAL_HOSTNAME-NAME` | ex `os-legal-dot-opendes.appspot.com` | Legal hostname| no | - |
+| `GCLOUD_PROJECT` | ex `osdu-cicd-epam` | Google cloud project id | no | -- |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `RECORDS_ROOT_URL` | ex `https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2` / Storage API endpoint | no | output of infrastructure deployment |
+| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_STORAGE_HOST` | ex `127.0.0.1` | Redis host for storage | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_PORT` | ex `6379` | Redis port for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_STORAGE_PORT` | ex `6379` | Redis port for storage | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
+| `ENABLE_FULL_BUCKET_NAME` | ex `true` | Full bucket name | no | - |
+
 Check that maven is installed:
 
 ```bash
@@ -126,8 +146,7 @@ You will need to have the following environment variables defined.
 | `HOST_URL` | `http://localhsot:8080/api/legal/v1/` | - | yes | - |
 | `MY_TENANT` | `osdu` | OSDU tenant used for testing | yes | - |
 | `SKIP_HTTP_TESTS` | ex `true` | jetty server returns 403 when running locally when deployed jettyserver is not used and the app returns a 302 so just run against deployed version only when checking http -> https redirects. Use 'true' for Google Cloud Run | yes | - |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
+| `ENABLE_FULL_BUCKET_NAME` | ex `true` | Full bucket name | no | - |
 
 **Entitlements configuration for integration accounts**
 
@@ -174,4 +193,4 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
+limitations under the License.

provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/security/GSuiteSecurityConfiguration.java

@@ -41,6 +41,7 @@ public class GSuiteSecurityConfiguration extends WebSecurityConfigurerAdapter {
   public void configure(WebSecurity web) throws Exception {
     web.ignoring().antMatchers("/api-docs")
         .antMatchers("/index")
+        .antMatchers("/info")
         .antMatchers("/swagger");
   }
 

provider/legal-gcp/src/main/resources/application.properties

@@ -11,4 +11,4 @@ JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupanc
 enable.full.bucket.name=false
 
 google.audiences=123.apps.googleusercontent.com
-partition.api=http://localhost:8081/api/partition/v1
\ No newline at end of file
+partition.api=http://localhost:8081/api/partition/v1