Legal cron job community

provider/legal-azure/README.md

@@ -79,6 +79,8 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
 | `AZURE_LEGAL_TOPICNAME` | `legaltags` | Same as `servicebus_topic_name` above | no | - |
 | `HOST_URL` | `http://localhost:8080/` | local service endpoint | no | - |
 | `ENTITLEMENT_URL` | Same as `entitlements_service_endpoint` above | Endpoint for entitlements URL | no | - |
+| `MY_SECOND_TENANT` | same as `opendes` | OSDU tenant used for testing | no | -- |
+| `AZURE_LEGAL_STORAGE_ACCOUNT_SECOND` | same as `storage_account` above | Storage account for integration tests| no | - |
 
 
 ### Configure Maven
@@ -170,6 +172,7 @@ export AZURE_SERVICE="legal"
 export AZURE_BUILD_SUBDIR="provider/legal-azure"
 export AZURE_TEST_SUBDIR="testing/legal-test-azure"
 export AZURE_OSDU_TENANT="opendes"
+export AZURE_OSDU_SECOND_TENANT="opendes2"
 export AZURE_SERVICE_BUS_TOPIC="legaltags"
 export LEGAL_STORAGE_CONTAINER="legal-service-azure-configuration"
 
@@ -185,11 +188,13 @@ export AZURE_APPSERVICE_NAME="${AZURE_BASENAME_21}-au-${AZURE_SERVICE}"
 export HOST_URL="https://${AZURE_BASENAME_21}-au-legal.azurewebsites.net/"
 export ENTITLEMENT_URL="https://${AZURE_BASENAME_21}-au-entitlements.azurewebsites.net/"
 export MY_TENANT="${AZURE_OSDU_TENANT}"
+export MY_SECOND_TENANT="${AZURE_OSDU_SECOND_TENANT}"
 export AZURE_AD_TENANT_ID="${AZURE_TENANT_ID}"
 export INTEGRATION_TESTER="${AZURE_PRINCIPAL_ID}"
 export AZURE_TESTER_SERVICEPRINCIPAL_SECRET="${AZURE_PRINCIPAL_SECRET}"
 export AZURE_AD_APP_RESOURCE_ID="${AZURE_APP_ID}"
 export AZURE_LEGAL_STORAGE_ACCOUNT="${AZURE_BASE}sa"
+export AZURE_LEGAL_STORAGE_ACCOUNT_SECOND="${AZURE_BASE}sa2"
 export AZURE_LEGAL_STORAGE_KEY="${AZURE_STORAGE_KEY}"
 export AZURE_LEGAL_SERVICEBUS="Endpoint=sb://${AZURE_BASENAME_21}sb.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=${AZURE_SERVICEBUS_KEY}"
 export AZURE_LEGAL_TOPICNAME="${AZURE_SERVICE_BUS_TOPIC}"

testing/legal-test-aws/src/test/java/org/opengroup/osdu/legal/util/AwsLegalTagUtils.java

@@ -74,6 +74,11 @@ public class AwsLegalTagUtils extends LegalTagUtils {
         return "Bearer " + BearerToken;
     }
 
+    @Override
+    public List<String> readCOOCountries(String storageAccount, String defaultCOOFileName) throws IOException {
+        return List.of();
+    }
+
     public void insertExpiredLegalTag() {
         // directly create expired legal tag document
         String integrationTagTestName = String.format("%s-dps-integration-test-1566474656479", getMyDataPartition()); // name has to match what's hardcoded in the test

testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/acceptanceTests/TestCronJobUpdate.java

+//  Copyright © SLB
+//  Copyright © Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.opengroup.osdu.legal.acceptanceTests;
+
+import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.opengroup.osdu.legal.util.AzureLegalTagUtils;
+import org.opengroup.osdu.legal.util.TestUtils;
+
+import java.util.List;
+
+
+public class TestCronJobUpdate extends CronJobUpdateTests {
+
+    private static final String storageAccount1 = System.getProperty("AZURE_LEGAL_STORAGE_ACCOUNT", System.getenv("AZURE_LEGAL_STORAGE_ACCOUNT"));
+    private static final String storageAccount2 = System.getProperty("AZURE_LEGAL_STORAGE_ACCOUNT_SECOND", System.getenv("AZURE_LEGAL_STORAGE_ACCOUNT_SECOND"));
+
+    private static final String defaultCOOJSONFileName1 = "TenantConfigTestingPurpose.json";
+    private static final String defaultCOOJSONFileName2 = "SecondTenantConfigTestingPurpose.json";
+
+    @Before
+    public void setup() throws Exception {
+        this.legalTagUtils = new AzureLegalTagUtils();
+    }
+
+    @After
+    public void teardown() throws Exception {
+        this.legalTagUtils = null;
+    }
+
+    @Test
+    public void should_returnOk_when_runCronJob() throws Exception {
+
+        String secondPartition = TestUtils.getMySecondDataPartition();
+        Assume.assumeFalse(secondPartition == null || storageAccount1 == null || storageAccount2 == null);
+
+        List<String> coo1 = legalTagUtils.readCOOCountries(storageAccount1, defaultCOOJSONFileName1);
+        List<String> coo2 = legalTagUtils.readCOOCountries(storageAccount2, defaultCOOJSONFileName2);
+
+        Assume.assumeTrue(coo1 != null && !coo1.isEmpty());
+        this.firstCountry = coo1.get(0);
+        Assume.assumeFalse(firstCountry == null);
+        this.secondCountry = coo2.stream().filter(e -> !e.equals(coo1.get(0))).findFirst().orElse(null);
+        Assume.assumeFalse(secondCountry == null);
+
+        super.should_returnOk_when_runCronJob();
+    }
+}

testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureLegalTagUtils.java

@@ -15,7 +15,10 @@
 package org.opengroup.osdu.legal.util;
 
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import com.azure.identity.ClientSecretCredential;
 import com.azure.identity.ClientSecretCredentialBuilder;
@@ -23,6 +26,8 @@ import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobContainerClientBuilder;
 import com.azure.storage.blob.BlobUrlParts;
 import com.azure.storage.blob.specialized.BlockBlobClient;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Strings;
 import org.opengroup.osdu.azure.util.AzureServicePrincipal;
 
@@ -59,6 +64,69 @@ public class AzureLegalTagUtils extends LegalTagUtils {
         }
     }
 
+    @Override
+    public List<String> readCOOCountries(String storageAccount, String defaultCOOFileName) throws IOException {
+
+        List<String> countries = new ArrayList<>();
+
+        String blobPath = String.format("https://%s.blob.core.windows.net/%s/%s", storageAccount.toLowerCase(), CONTAINER_NAME_AZURE, FILE_NAME);
+        BlobUrlParts parts = BlobUrlParts.parse(blobPath);
+        ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder()
+                .clientSecret(clientSecret)
+                .clientId(clientId)
+                .tenantId(tenantId)
+                .build();
+        BlobContainerClient blobContainerClient = new BlobContainerClientBuilder()
+                .endpoint(String.format("https://%s.blob.core.windows.net", parts.getAccountName()))
+                .credential(clientSecretCredential)
+                .containerName(parts.getBlobContainerName())
+                .buildClient();
+
+        BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(parts.getBlobName()).getBlockBlobClient();
+        if (blobContainerClient.exists()) {
+            try {
+                if (blockBlobClient.exists()) {
+                    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+                    blockBlobClient.download(outputStream);
+                    ByteArrayInputStream stream = new ByteArrayInputStream(outputStream.toByteArray());
+                    int byteData;
+                    StringBuffer sb = new StringBuffer();
+                    while ((byteData = stream.read()) != -1) {
+                        sb.append((char) byteData);
+                    }
+
+                    ObjectMapper objectMapper = new ObjectMapper();
+                    JsonNode root = objectMapper.readTree(sb.toString());
+
+                    if(root.isArray()) {
+                        for (JsonNode node : root) {
+                            countries.add(node.path("alpha2").toString().replaceAll("\"", ""));
+                        }
+                    }
+                    stream.close();
+                } else {
+                    String content = readTestFile(defaultCOOFileName);
+                    ByteArrayInputStream newStream = new ByteArrayInputStream(content.getBytes());
+                    newStream.reset();
+                    blockBlobClient.upload(newStream, content.length(), true);
+
+                    ObjectMapper objectMapper = new ObjectMapper();
+                    JsonNode root = objectMapper.readTree(content);
+
+                    if(root.isArray()) {
+                        for (JsonNode node : root) {
+                            countries.add(node.path("alpha2").toString().replaceAll("\"", ""));
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                throw new AssertionError(String.format("Error: Could not create test %s file blob", parts.getBlobName()), e);
+            }
+        }
+
+        return countries;
+    }
+
     private static String generateContainerPath(String accountName, String containerName) {
         return String.format("https://%s.blob.core.windows.net/%s", accountName, containerName);
     }

testing/legal-test-baremetal/src/test/java/legal/util/AnthosLegalTagUtils.java

@@ -21,6 +21,9 @@ import com.google.common.base.Strings;
 import lombok.extern.slf4j.Slf4j;
 import org.opengroup.osdu.legal.util.LegalTagUtils;
 
+import java.io.IOException;
+import java.util.List;
+
 @Slf4j
 public class AnthosLegalTagUtils extends LegalTagUtils {
 
@@ -40,4 +43,9 @@ public class AnthosLegalTagUtils extends LegalTagUtils {
         }
         return "Bearer " + token;
     }
+
+    @Override
+    public List<String> readCOOCountries(String storageAccount, String defaultCOOFileName) throws IOException {
+        return List.of();
+    }
 }

testing/legal-test-core/src/main/java/org/opengroup/osdu/legal/acceptanceTests/CronJobUpdateTests.java

+//  Copyright © SLB
+//  Copyright © Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.opengroup.osdu.legal.acceptanceTests;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.opengroup.osdu.legal.util.Constants;
+import org.opengroup.osdu.legal.util.LegalTagUtils;
+import org.opengroup.osdu.legal.util.TestUtils;
+
+import java.util.Map;
+
+public abstract class CronJobUpdateTests {
+
+    protected LegalTagUtils legalTagUtils;
+
+    private final static String LEGAL_TAG_DATE = "2099-12-25";
+    private final static String LEGAL_TAG_TYPE = "Third Party Data";
+    private final static String LEGAL_TAG_DESC = "description";
+
+    protected String firstCountry;
+    protected String secondCountry;
+
+    @Test
+    public void should_returnOk_when_runCronJob() throws Exception {
+
+        String primaryPartition = TestUtils.getMyDataPartition();
+        String secondPartition = TestUtils.getMySecondDataPartition();
+
+        Assume.assumeFalse(secondPartition == null || firstCountry == null || secondCountry == null);
+
+        String accessToken = this.legalTagUtils.accessToken();
+
+        Map<String, String> headers = this.legalTagUtils.getHeaders();
+        headers.put(Constants.DATA_PARTITION_ID, primaryPartition);
+        String tagName = LegalTagUtils.createRandomNameTenant();
+
+        LegalTagUtils.ReadableLegalTag legalTag1Ret = creteLegalTag(firstCountry, tagName, primaryPartition);
+        LegalTagUtils.ReadableLegalTag legalTag2Ret = creteLegalTag(secondCountry, tagName, secondPartition);
+
+        runCronJob(headers, accessToken);
+
+        headers.put(Constants.DATA_PARTITION_ID, secondPartition);
+        runCronJob(headers, accessToken);
+
+        validateTag(primaryPartition, tagName, accessToken, firstCountry, legalTag1Ret);
+        validateTag(secondPartition, tagName, accessToken, secondCountry, legalTag2Ret);
+
+        this.legalTagUtils.delete(tagName, primaryPartition);
+        this.legalTagUtils.delete(tagName, secondPartition);
+    }
+
+    private LegalTagUtils.ReadableLegalTag creteLegalTag(String country, String tagName, String partition) throws Exception {
+        ClientResponse response = this.legalTagUtils.create(country, tagName, LEGAL_TAG_DATE, LEGAL_TAG_TYPE, partition, LEGAL_TAG_DESC);
+        Assert.assertEquals(201, response.getStatus());
+
+        return this.legalTagUtils.getResult(response, 201, LegalTagUtils.ReadableLegalTag.class);
+    }
+
+    private void runCronJob(Map<String, String> headers, String accessToken) throws Exception {
+        ClientResponse response = this.legalTagUtils.send("jobs/updateLegalTagStatus",
+                "GET", accessToken, null, "", headers);
+        Assert.assertEquals(204, response.getStatus());
+    }
+
+    private void validateTag(String partitionId, String tagName, String accessToken, String cco, LegalTagUtils.ReadableLegalTag legalTagRet) throws Exception {
+        Map<String, String> headers = this.legalTagUtils.getHeaders();
+        headers.put(Constants.DATA_PARTITION_ID, partitionId);
+        ClientResponse response = this.legalTagUtils.send("legaltags/" + tagName, "GET", accessToken, null, "", headers);
+        LegalTagUtils.ReadableLegalTag legalTag1  = this.legalTagUtils.getResult(response, 200, LegalTagUtils.ReadableLegalTag.class);
+
+        String[] countryParam = {cco};
+        Assert.assertEquals(legalTag1.properties.countryOfOrigin, countryParam);
+        Assert.assertEquals(legalTag1.properties.originator, "MyCompany");
+        Assert.assertEquals(legalTag1.properties.dataType, "Third Party Data");
+        Assert.assertEquals(legalTag1.properties.securityClassification, "Public");
+        Assert.assertEquals(legalTag1.properties.personalData, "No Personal Data");
+        Assert.assertEquals(legalTag1.properties.exportClassification, "EAR99");
+
+        // Validate legal tag
+        response = this.legalTagUtils.send("legaltags:validate", "POST", accessToken, LegalTagUtils.createRetrieveBatchBody(legalTagRet.name), "", headers);
+        LegalTagUtils.InvalidTagsWithReason invalidTagsWithReason = this.legalTagUtils.getResult(response, 200, LegalTagUtils.InvalidTagsWithReason.class);
+        Assert.assertEquals(0, invalidTagsWithReason.invalidLegalTags.length);
+    }
+}

testing/legal-test-core/src/main/java/org/opengroup/osdu/legal/util/LegalTagUtils.java

@@ -7,6 +7,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -25,6 +26,8 @@ public abstract class LegalTagUtils extends TestUtils {
         return LegalTagUtils.class.getResourceAsStream("/" + fileName);
     }
 
+    public abstract List<String> readCOOCountries(String storageAccount, String defaultCOOFileName) throws IOException;
+
     protected static String readTestFile(String fileName) throws IOException {
         InputStream inputStream = getTestFileInputStream(fileName);
         if(inputStream == null) {

testing/legal-test-core/src/main/java/org/opengroup/osdu/legal/util/TestUtils.java

@@ -61,6 +61,10 @@ public class TestUtils {
 		return System.getProperty("MY_TENANT", System.getenv("MY_TENANT"));
 	}
 
+	public static String getMySecondDataPartition(){
+		return System.getProperty("MY_SECOND_TENANT", System.getenv("MY_SECOND_TENANT"));
+	}
+
 	public ClientResponse send(String path, String httpMethod, String token, String requestBody, String query)
 			throws Exception {
 

testing/legal-test-core/src/main/resources/SecondTenantConfigTestingPurpose.json

+[{
+  "name": "Malaysia",
+  "alpha2": "MY",
+  "numeric": 458,
+  "residencyRisk": "Client consent required"
+}, {
+  "name": "Azerbaijan",
+  "alpha2": "AZ",
+  "numeric": 31,
+  "residencyRisk": "Client consent required",
+  "typesNotApplyDataResidency": ["Transferred Data"]
+}]
\ No newline at end of file

testing/legal-test-gc/src/test/java/org/opengroup/osdu/legal/util/GCPLegalTagUtils.java

@@ -20,6 +20,9 @@ package org.opengroup.osdu.legal.util;
 import com.google.api.client.util.Strings;
 import lombok.extern.slf4j.Slf4j;
 
+import java.io.IOException;
+import java.util.List;
+
 @Slf4j
 public class GCPLegalTagUtils extends LegalTagUtils {
 
@@ -35,4 +38,9 @@ public class GCPLegalTagUtils extends LegalTagUtils {
     }
     return "Bearer " + token;
   }
+
+  @Override
+  public List<String> readCOOCountries(String storageAccount, String defaultCOOFileName) throws IOException {
+    return List.of();
+  }
 }

testing/legal-test-ibm/src/test/java/org/opengroup/osdu/legal/util/IBMLegalTagUtils.java

@@ -19,6 +19,9 @@ import org.opengroup.osdu.core.ibm.util.IdentityClient;
 
 import com.google.common.base.Strings;
 
+import java.io.IOException;
+import java.util.List;
+
 public class IBMLegalTagUtils extends LegalTagUtils {
 
     @Override
@@ -33,7 +36,12 @@ public class IBMLegalTagUtils extends LegalTagUtils {
         }
         return "Bearer " + token;
     }
-    
+
+    @Override
+    public List<String> readCOOCountries(String storageAccount, String defaultCOOFileName) throws IOException {
+        return List.of();
+    }
+
     private static final String mqConnection = "IBM_LEGAL_MQ_CONNECTION";
     private static final String envPrefix = "IBM_ENV_PREFIX";