Skip to content
Snippets Groups Projects
Commit 1a113ecf authored by Konrad Krasnodebski's avatar Konrad Krasnodebski
Browse files

Refactor about to expire checking

parent 1320f6a2
No related branches found
No related tags found
2 merge requests!620Update version of default branch to 0.27.0-SNAPSHOT,!535Extend cron to catch about to expire legal tags
Hide whitespace changes
Inline Side-by-side
Showing
with 129 additions and 109 deletions
devops/core-plus/deploy/templates/cronjob.yaml
+ 1
− 0
View file @ 1a113ecf
......@@ -4,6 +4,7 @@ metadata:
name: {{ printf "%s-legalstatus-update" .Values.conf.appName | quote }}
namespace: {{ .Release.Namespace | quote }}
spec:
# when cron runs less frequent than once per a day it could not match when legal tag be about to expire!
schedule: "@daily"
jobTemplate:
spec:
......
docs/tutorial/ComplianceService.md
+ 7
− 0
View file @ 1a113ecf
......@@ -544,6 +544,13 @@ If it has become incompliant, you must make sure associated data is no longer al
If it is marked compliant, data that was not allowed for consumption can now be consumed through your services.
Time to expire should be configurable on deployment level. We should be able to provide list of all time periods before LegalTag expires. This will be achieved by introducing new environment variable `legaltag.expirationAlerts`. Format is comma separated values which consists of number and letter suffix, for example: 3d,2w,1m
Where suffix letter should represent this time periods:
* #d - # number of days
* #w - # number of weeks
* #m - # number of months
[Back to table of contents](#TOC)
## Version info endpoint
......
legal-core/src/main/java/org/opengroup/osdu/legal/ClockConfig.java 0 → 100644
+ 15
− 0
View file @ 1a113ecf
package org.opengroup.osdu.legal;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.time.Clock;
@Configuration
public class ClockConfig {
@Bean
Clock clock() {
return Clock.systemDefaultZone();
}
}
legal-core/src/main/java/org/opengroup/osdu/legal/jobs/LegalTagStatusJob.java
+ 55
− 32
View file @ 1a113ecf
......@@ -17,15 +17,24 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.legal.FeatureFlagController;
import org.springframework.stereotype.Component;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.time.Clock;
import java.time.LocalDate;
import java.time.ZoneId;
import jakarta.inject.Inject;
@Component
@ConfigurationProperties("legaltag")
public class LegalTagStatusJob {
@Inject
private LegalTagConstraintValidator validator;
......@@ -40,8 +49,10 @@ public class LegalTagStatusJob {
@Inject
private FeatureFlagController featureFlagController;
@Value("${LEGALTAG_EXPIRATION:}")
private String legalTagExpiration;
@Autowired
private Clock clock;
private String expirationAlerts;
public LegalTagJobResult run(String projectId, DpsHeaders headers, String tenantName) throws Exception {
LegalTagJobResult legalTagJobResult = new LegalTagJobResult(new StatusChangedTags(), new AboutToExpireLegalTags());
......@@ -81,42 +92,54 @@ public class LegalTagStatusJob {
private void checkAboutToExpireLegalTag(LegalTag tag, AboutToExpireLegalTags aboutToExpireLegalTags) {
Properties properties = tag.getProperties();
Date expirationDate = properties.getExpirationDate();
Date today = new Date();
Date aboutToExpireDate = getAboutToExpireDate(expirationDate);
Boolean isNotAboutToExpire = aboutToExpireDate.after(today);
if (!isNotAboutToExpire) {
log.info(String.format("Found legal tag about to expire: %s", tag.getName()));
aboutToExpireLegalTags.getAboutToExpireLegalTags().add(new AboutToExpireLegalTag(tag.getName(), properties.getExpirationDate()));
List<LocalDate> aboutToExpireDates = getAboutToExpireDates(expirationDate);
LocalDate today = LocalDate.now(clock);
for (LocalDate aboutToExpireDate : aboutToExpireDates) {
// when cron runs less frequent than once per a day it could not match when legal tag be about to expire!
Boolean isAboutToExpire = aboutToExpireDate.equals(today);
if (isAboutToExpire) {
log.info(String.format("Found legal tag about to expire: %s at %s", tag.getName(), expirationDate));
aboutToExpireLegalTags.getAboutToExpireLegalTags().add(new AboutToExpireLegalTag(tag.getName(), properties.getExpirationDate()));
return;
}
}
}
private Date getAboutToExpireDate(Date expirationDate) throws AppException {
Calendar cal = Calendar.getInstance();
cal.setTime(expirationDate);
try {
if (legalTagExpiration.contains("d")) {
int numberOfDays = Integer.parseInt(legalTagExpiration.replace("d", ""));
cal.add(Calendar.DAY_OF_YEAR, - numberOfDays);
} else if (legalTagExpiration.contains("w")) {
int numberOfWeeks = Integer.parseInt(legalTagExpiration.replace("w", ""));
cal.add(Calendar.DAY_OF_YEAR, - 7 * numberOfWeeks);
} else if (legalTagExpiration.contains("m")) {
int numberOfMonths = Integer.parseInt(legalTagExpiration.replace("m", ""));
cal.add(Calendar.MONTH, - numberOfMonths);
} else if (legalTagExpiration.contains("y")) {
int numberOfYears = Integer.parseInt(legalTagExpiration.replace("y", ""));
cal.add(Calendar.YEAR, - numberOfYears);
} else {
throw new AppException(500, "Server error", String.format("Invalid legalTagExpiration value: %s", legalTagExpiration));
private List<LocalDate> getAboutToExpireDates(Date expirationDate) throws AppException {
ArrayList aboutToExpireDates = new ArrayList<>();
List<String> expirationAlertsList = Arrays.asList(expirationAlerts.split(","));
for (String expiration : expirationAlertsList) {
Calendar cal = Calendar.getInstance();
cal.setTime(expirationDate);
try {
if (expiration.contains("d")) {
int numberOfDays = Integer.parseInt(expiration.replace("d", ""));
cal.add(Calendar.DAY_OF_YEAR, - numberOfDays);
} else if (expiration.contains("w")) {
int numberOfWeeks = Integer.parseInt(expiration.replace("w", ""));
cal.add(Calendar.DAY_OF_YEAR, - 7 * numberOfWeeks);
} else if (expiration.contains("m")) {
int numberOfMonths = Integer.parseInt(expiration.replace("m", ""));
cal.add(Calendar.MONTH, - numberOfMonths);
} else if (expiration.contains("y")) {
int numberOfYears = Integer.parseInt(expiration.replace("y", ""));
cal.add(Calendar.YEAR, - numberOfYears);
} else {
log.error(String.format("Invalid legal tag about to expire time value: %s", expiration));
throw new AppException(500, "Server error", String.format("Invalid legal tag about to expire time value: %s", expiration));
}
} catch (NumberFormatException e) {
log.error(String.format("Invalid legal tag about to expire time value: %s", expiration));
throw new AppException(500, "Server error", String.format("Invalid legal tag about to expire time value: %s", expiration));
}
} catch (NumberFormatException e) {
log.error(String.format("Invalid legalTagExpiration value: %s", legalTagExpiration));
throw new AppException(500, "Server error", String.format("Invalid legalTagExpiration value: %s", legalTagExpiration));
aboutToExpireDates.add(cal.getTime().toInstant().atZone(ZoneId.systemDefault()).toLocalDate());
}
return cal.getTime();
return aboutToExpireDates;
}
private void publishLegalTagStatusUpdateEvents(boolean hasStatusChanges, String projectId, DpsHeaders headers, StatusChangedTags statusChangedTags) throws Exception {
......
legal-core/src/main/resources/application.properties
+ 2
− 1
View file @ 1a113ecf
legaltag.expirationAlerts=1m,2w,1d
# Feature flag settings
featureFlag.strategy=appProperty
featureFlag.aboutToExpireLegalTag.enabled=false
\ No newline at end of file
featureFlag.aboutToExpireLegalTag.enabled=false
legal-core/src/test/java/org/opengroup/osdu/legal/jobs/LegalTagStatusJobTests.java
+ 49
− 74
View file @ 1a113ecf
......@@ -27,6 +27,10 @@ import org.springframework.test.util.ReflectionTestUtils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Calendar;
import java.time.Clock;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
......@@ -50,12 +54,18 @@ public class LegalTagStatusJobTests {
private JaxRsDpsLog log;
@Mock
private FeatureFlagController featureFlagControllerMock;
@Mock
private Clock clock;
@InjectMocks
LegalTagStatusJob sut;
private Clock fixedClock;
private DpsHeaders headers = new DpsHeaders();
private final static LocalDate LOCAL_DATE = LocalDate.of(2022, 01, 01);
@Before
public void setup() {
when(validator.getErrors(any())).thenReturn(null);
......@@ -64,7 +74,11 @@ public class LegalTagStatusJobTests {
headers.put(DpsHeaders.USER_EMAIL, "nonexistent@nonexisent.domain");
// aboutToExpireFeatureFlag
when(featureFlagControllerMock.isAboutToExpireFeatureFlagEnabled()).thenReturn(true);
ReflectionTestUtils.setField(sut, "legalTagExpiration", "1d");
ReflectionTestUtils.setField(sut, "expirationAlerts", "1m,2w,1d");
// mock LocalDate.now()
fixedClock = Clock.fixed(LOCAL_DATE.atStartOfDay(ZoneId.systemDefault()).toInstant(), ZoneId.systemDefault());
doReturn(fixedClock.instant()).when(clock).instant();
doReturn(fixedClock.getZone()).when(clock).getZone();
}
@Test
......@@ -142,137 +156,98 @@ public class LegalTagStatusJobTests {
}
@Test
public void should_returnAboutToExpireLegalTagName_when_legalTagAboutToExpireIn3Days() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "3d");
public void should_returnAboutToExpireLegalTagName_when_legalTagAboutToExpireIn1Year1Month2Weeks1Day() throws Exception {
ReflectionTestUtils.setField(sut, "expirationAlerts", "1y,1m,2w,1d");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag aboutToExpireLegalTag1 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag1", true, 2);
LegalTag aboutToExpireLegalTag1 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag1", true, 365);
validLegalTags.add(aboutToExpireLegalTag1);
LegalTag aboutToExpireLegalTag2 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag2", true, 3);
LegalTag aboutToExpireLegalTag2 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag2", true, 31);
validLegalTags.add(aboutToExpireLegalTag2);
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("longTermLegalTag", true, 10);
validLegalTags.add(longTermLegalTag);
LegalTag aboutToExpireLegalTag3 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag3", true, 14);
validLegalTags.add(aboutToExpireLegalTag3);
LegalTag aboutToExpireLegalTag4 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag4", true, 1);
validLegalTags.add(aboutToExpireLegalTag4);
LegalTag anotherLegalTag1 = createValidLegalTagWithIsValidStatus("anotherLegalTag1", true, 2);
validLegalTags.add(anotherLegalTag1);
LegalTag anotherLegalTag2 = createValidLegalTagWithIsValidStatus("anotherLegalTag2", true, 30);
validLegalTags.add(anotherLegalTag2);
LegalTag anotherLegalTag3 = createValidLegalTagWithIsValidStatus("anotherLegalTag3", true, 100);
validLegalTags.add(anotherLegalTag3);
sut = createSut(validLegalTags);
LegalTagJobResult result = sut.run("project1", headers, "tenant");
assertEquals(2, result.aboutToExpireLegalTags.getAboutToExpireLegalTags().size());
assertEquals(4, result.aboutToExpireLegalTags.getAboutToExpireLegalTags().size());
assertEquals("aboutToExpireLegalTag1", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(0).getTagName());
assertEquals("aboutToExpireLegalTag2", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(1).getTagName());
assertEquals("aboutToExpireLegalTag3", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(2).getTagName());
assertEquals("aboutToExpireLegalTag4", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(3).getTagName());
}
@Test
public void should_returnAboutToExpireLegalTagName_when_legalTagAboutToExpireIn2Weeks() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "2w");
ReflectionTestUtils.setField(sut, "expirationAlerts", "2w");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag aboutToExpireLegalTag1 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag1", true, 13);
validLegalTags.add(aboutToExpireLegalTag1);
LegalTag aboutToExpireLegalTag2 = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag2", true, 14);
validLegalTags.add(aboutToExpireLegalTag2);
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("longTermLegalTag", true, 20);
validLegalTags.add(longTermLegalTag);
sut = createSut(validLegalTags);
LegalTagJobResult result = sut.run("project1", headers, "tenant");
assertEquals(2, result.aboutToExpireLegalTags.getAboutToExpireLegalTags().size());
assertEquals("aboutToExpireLegalTag1", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(0).getTagName());
assertEquals("aboutToExpireLegalTag2", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(1).getTagName());
}
@Test
public void should_returnAboutToExpireLegalTagName_when_legalTagAboutToExpireIn3Months() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "3m");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag aboutToExpireLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag", true, 80);
validLegalTags.add(aboutToExpireLegalTag);
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("longTermLegalTag", true, 100);
validLegalTags.add(longTermLegalTag);
LegalTag anotherLegalTag = createValidLegalTagWithIsValidStatus("anotherLegalTag", true, 20);
validLegalTags.add(anotherLegalTag);
sut = createSut(validLegalTags);
LegalTagJobResult result = sut.run("project1", headers, "tenant");
assertEquals(1, result.aboutToExpireLegalTags.getAboutToExpireLegalTags().size());
assertEquals("aboutToExpireLegalTag", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(0).getTagName());
assertEquals("aboutToExpireLegalTag2", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(0).getTagName());
}
@Test
public void should_returnAboutToExpireLegalTagName_when_legalTagAboutToExpireIn1Year() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "1y");
public void should_throwException_when_wrongExpirationTime() throws Exception {
ReflectionTestUtils.setField(sut, "expirationAlerts", "2d,XXXy");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag aboutToExpireLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag", true, 354);
validLegalTags.add(aboutToExpireLegalTag);
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("longTermLegalTag", true, 367);
validLegalTags.add(longTermLegalTag);
sut = createSut(validLegalTags);
LegalTagJobResult result = sut.run("project1", headers, "tenant");
assertEquals(1, result.aboutToExpireLegalTags.getAboutToExpireLegalTags().size());
assertEquals("aboutToExpireLegalTag", result.aboutToExpireLegalTags.getAboutToExpireLegalTags().get(0).getTagName());
}
@Test
public void should_throwException_when_legalTagBadExpireDate() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "XXXy");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag aboutToExpireLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag", true, 354);
validLegalTags.add(aboutToExpireLegalTag);
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("longTermLegalTag", true, 367);
validLegalTags.add(longTermLegalTag);
try {
sut = createSut(validLegalTags);
LegalTagJobResult result = sut.run("project1", headers, "tenant");
fail("Should throw an exception");
} catch (AppException e) {
assertEquals("Invalid legalTagExpiration value: XXXy", e.getMessage());
assertEquals("Invalid legal tag about to expire time value: XXXy", e.getMessage());
assertEquals(500, e.getError().getCode());
}
}
@Test
public void should_throwException_when_legalTagBadFormat() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "XXX");
public void should_throwException_when_wrongExpirationFormat() throws Exception {
ReflectionTestUtils.setField(sut, "expirationAlerts", "XXX");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag aboutToExpireLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag", true, 354);
validLegalTags.add(aboutToExpireLegalTag);
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("longTermLegalTag", true, 367);
validLegalTags.add(longTermLegalTag);
try {
sut = createSut(validLegalTags);
LegalTagJobResult result = sut.run("project1", headers, "tenant");
fail("Should throw an exception");
} catch (AppException e) {
assertEquals("Invalid legalTagExpiration value: XXX", e.getMessage());
assertEquals("Invalid legal tag about to expire time value: XXX", e.getMessage());
assertEquals(500, e.getError().getCode());
}
}
@Test
public void should_returnAboutToExpireLegalTagName_when_invalidTimeToExpire() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "asd");
Collection<LegalTag> validLegalTags = new ArrayList<>();
sut = createSut(validLegalTags);
try {
sut.run("project1", headers, "tenant");
} catch (Exception e) {
fail(String.format("Should not throw this exception: '%s'", e.getMessage()));
}
}
@Test
public void should_sendLegalTagNameToPubSub_when_legalTagAboutToExpire() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "2w");
ReflectionTestUtils.setField(sut, "expirationAlerts", "2w");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag aboutToExpireLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag1", true, 13);
LegalTag aboutToExpireLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag", true, 14);
validLegalTags.add(aboutToExpireLegalTag);
sut = createSut(validLegalTags);
......@@ -284,10 +259,10 @@ public class LegalTagStatusJobTests {
@Test
public void should_notSendLegalTagNameToPubSub_when_legalTagNotAboutToExpire() throws Exception {
ReflectionTestUtils.setField(sut, "legalTagExpiration", "2w");
ReflectionTestUtils.setField(sut, "expirationAlerts", "2w");
Collection<LegalTag> validLegalTags = new ArrayList<>();
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag1", true, 20);
LegalTag longTermLegalTag = createValidLegalTagWithIsValidStatus("aboutToExpireLegalTag", true, 20);
validLegalTags.add(longTermLegalTag);
sut = createSut(validLegalTags);
......@@ -303,7 +278,7 @@ public class LegalTagStatusJobTests {
legalTag.setIsValid(isValid);
if (daysToExpire > 0) {
Calendar cal = Calendar.getInstance();
cal.setTime(new java.util.Date(System.currentTimeMillis()));
cal.setTime(java.util.Date.from(LOCAL_DATE.atStartOfDay(ZoneId.systemDefault()).toInstant()));
cal.add(Calendar.DAY_OF_YEAR, + daysToExpire);
Properties properties = new Properties();
properties.setExpirationDate(new java.sql.Date(cal.getTimeInMillis()));
......
provider/legal-aws/src/main/resources/application.properties
+ 0
− 2
View file @ 1a113ecf
......@@ -31,8 +31,6 @@ LEGAL_HOSTNAME=notused
CRON_JOB_IP=10.0.0.1
ACCEPT_HTTP=true
LEGALTAG_EXPIRATION=2w
OSDU_TOPIC=${OSDU_LEGAL_TOPIC:legal-tag-status-changed}
OSDU_ABOUT_TO_EXPIRE_LEGALTAG_TOPIC=${OSDU_LEGAL_TOPIC:about-to-expire-legal-tag}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment