There is a security vulnerability in SSH key-generation using GitKraken < v8.0.1. If you used this tool to create SSH keys, please update GitKraken and regenerate. If you need help with this, contact forum-support@opengroup.org

Commit 4d414d4e authored by ethiraj krishnamanaidu's avatar ethiraj krishnamanaidu
Browse files

Merge branch 'ado-branch' into 'master'

ADO latest code merge

See merge request !2
parents 62393b3d 0b798e8c
Pipeline #2604 failed with stages
in 10 minutes and 32 seconds
......@@ -24,19 +24,19 @@ The Schema Service is a Maven multi-module project with each cloud implemention
server.port=8080
```
## Running Automated (DevSanity) Tests
## Running Automated Integration Test
DevSanity tests are located in a schema-core project in testing directory under the project root directory.
1. GCP
These tests validate functionality of schema service.
They can then be run/debugged directly in your IDE of choice using the GUI or via the commandline using below command from GCP project.
They can then be run/debugged directly in your IDE of choice using the GUI or via the commandline using below command from schema-core project.
Below command has to be run post building complete project.
cd provider\schema-gcp
mvn verify -P DevSanity
cd schema-core
mvn verify -P IntegrationTest
......
......@@ -58,25 +58,43 @@ stages:
targetType: 'inline'
script: |
#!/bin/bash
set -e
pushd $(dockerDir)
cat $(gcrKey.secureFilePath) | docker login -u _json_key --password-stdin https://gcr.io
echo $(dockerImageName)
docker-compose build $(dockerImageName)
docker tag gcr.io/opendes/$(dockerImageName) gcr.io/opendes/$(dockerImageName):$(tag)
docker push gcr.io/opendes/$(dockerImageName):$(tag)
docker push gcr.io/opendes/$(dockerImageName)
echo 'Push done.'
kubectl --kubeconfig $(kubeconfig.secureFilePath) rollout restart deployment/$(deploymentName)
popd
sleep 10
OUTPUT="Alive"
ENDPOINT=https://open.opendes.cloud.slb-ds.com/api/schema-service/v1/health
echo $ENDPOINT
while [ -z "$STATUS" ]; do
STATUS=`curl -v --silent --http1.0 "$ENDPOINT" 2>&1 | grep "$OUTPUT"`
echo $STATUS
if [ -z "$STATUS" ]; then
echo "Endpoint is not up yet."
sleep 10
else
echo "Endpoint is up"
fi
done
condition: succeeded()
displayName: 'build,upload and deploy docker image'
- task: Maven@3
displayName: 'DevSanity'
displayName: 'Running IntegrationTest'
inputs:
mavenPomFile: 'provider/$(osProjectName)-gcp/pom.xml'
mavenPomFile: '$(osProjectName)-core/pom.xml'
goals: 'verify'
options: '-P DevSanity --settings maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
options: '-P IntegrationTest --settings maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
publishJUnitResults: false
javaHomeOption: 'JDKVersion'
mavenVersionOption: 'Default'
......@@ -91,63 +109,6 @@ stages:
ArtifactName: 'drop'
publishLocation: 'Container'
condition: succeededOrFailed()
- stage: Deploy_DEV
dependsOn: Build
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
jobs:
- job: deploy
pool:
name: Hosted Ubuntu 1604
steps:
- task: DownloadSecureFile@1
name: gcrKey
inputs:
secureFile: 'gcr-push-key-file.json'
- task: DownloadSecureFile@1
name: kubeconfigs
inputs:
secureFile: 'kubeconfig'
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'specific'
project: '9bf2f6e3-4cfe-416c-80df-404767279175'
pipeline: '172'
buildVersionToDownload: 'latest'
downloadType: 'specific'
downloadPath: '$(System.ArtifactsDirectory)'
- task: Bash@3
inputs:
targetType: 'inline'
script: |
#!/bin/bash
set -e
pushd $(dockerDir)
cat $(gcrKey.secureFilePath) | docker login -u _json_key --password-stdin https://gcr.io
docker pull gcr.io/opendes/$(dockerImageName):$(tag)
docker tag gcr.io/opendes/$(dockerImageName):$(tag) gcr.io/opendes/$(dockerImageName)
docker push gcr.io/opendes/$(dockerImageName)
kubectl --kubeconfig $(kubeconfigs.secureFilePath) rollout restart deployment/$(deploymentName)
popd
displayName: 'deploy service'
- task: Maven@3
displayName: 'DevSanity'
inputs:
mavenPomFile: 'provider/schema-gcp/pom.xml'
goals: 'verify'
options: '-P DevSanity --settings maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
publishJUnitResults: false
javaHomeOption: 'JDKVersion'
mavenVersionOption: 'Default'
mavenAuthenticateFeed: false
effectivePomSkip: false
sonarQubeRunAnalysis: false
- stage: DeployToQA
condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
......
......@@ -59,14 +59,6 @@
<version>0.1.15</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-core</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java8</artifactId>
......@@ -147,95 +139,15 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M3</version>
<configuration>
<testFailureIgnore>true</testFailureIgnore>
<!-- <includes>
<include>*Test.java</include>
</includes> -->
<skipTests>${skipUnitTests}</skipTests>
<forkCount>3</forkCount>
<reuseForks>true</reuseForks>
<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.0.0-M3</version>
<executions>
<execution>
<id>pre-integration-test</id>
<goals>
<goal>integration-test</goal>
</goals>
<configuration>
<skipTests>${skipItTests}</skipTests>
<includes>
<include>org.opengroup.osdu.schema.impl.sanityTestRunner.TriggerPreIntegrationTestsRunner</include>
</includes>
</configuration>
</execution>
<execution>
<id>integration-test</id>
<goals>
<goal>integration-test</goal>
</goals>
<configuration>
<skipTests>${skipItTests}</skipTests>
<excludes>
<exclude>none</exclude>
</excludes>
<includes>
<include>org.opengroup.osdu.schema.impl.sanityTestRunner.TriggerSchemaServiceTestsRunner</include>
</includes>
</configuration>
</execution>
<execution>
<id>post-integration-test</id>
<goals>
<goal>integration-test</goal>
</goals>
<configuration>
<skipTests>${skipItTests}</skipTests>
<includes>
<include>org.opengroup.osdu.schema.impl.sanityTestRunner.TriggerPostIntegrationTestsRunner</include>
</includes>
</configuration>
</execution>
<execution>
<id>verify</id>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>Default</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<skipUnitTests>false</skipUnitTests>
<skipItTests>true</skipItTests>
<preIntegrationPhase>none</preIntegrationPhase>
<postIntegrationPhase>none</postIntegrationPhase>
<mainClass>org.opengroup.osdu.schema.SchemaApplication</mainClass>
</properties>
</profile>
<profile>
<id>DevSanity</id>
<properties>
<skipUnitTests>true</skipUnitTests>
<skipItTests>false</skipItTests>
<preIntegrationPhase>pre-integration-test</preIntegrationPhase>
<postIntegrationPhase>post-integration-test</postIntegrationPhase>
<mainClass>org.opengroup.osdu.schema.SchemaApplication</mainClass>
</properties>
</profile>
</profiles>
</project>
\ No newline at end of file
......@@ -81,9 +81,9 @@ public class GoogleEntityTypeStore implements IEntityTypeStore {
datastore.add(entity);
} catch (DatastoreException ex) {
if ("ALREADY_EXISTS".equals(ex.getReason())) {
log.warning(SchemaConstants.ENTITY_EXISTS);
log.warning(SchemaConstants.ENTITY_TYPE_EXISTS);
throw new BadRequestException(
MessageFormat.format(SchemaConstants.ENTITY_EXISTS_EXCEPTION, entityType.getEntityTypeId()));
MessageFormat.format(SchemaConstants.ENTITY_TYPE_EXISTS_EXCEPTION, entityType.getEntityTypeId()));
} else {
log.severe(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage()));
throw new ApplicationException("Invalid input, object invalid");
......
......@@ -168,7 +168,7 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
.setKind(SchemaConstants.SCHEMA_KIND)
.setFilter(CompositeFilter.and(
PropertyFilter.eq(SchemaConstants.AUTHORITY, schemaInfo.getSchemaIdentity().getAuthority()),
PropertyFilter.eq(SchemaConstants.ENTITY_TYPE, schemaInfo.getSchemaIdentity().getEntity()),
PropertyFilter.eq(SchemaConstants.ENTITY_TYPE, schemaInfo.getSchemaIdentity().getEntityType()),
PropertyFilter.eq(SchemaConstants.MAJOR_VERSION,
schemaInfo.getSchemaIdentity().getSchemaVersionMajor()),
PropertyFilter.eq(SchemaConstants.SOURCE, schemaInfo.getSchemaIdentity().getSource())))
......@@ -191,7 +191,7 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
SchemaIdentity superSededBy = null;
if (entity.contains(SchemaConstants.SUPERSEDED_BY)) {
KeyFactory keyFactory = datastore.newKeyFactory().setKind(SchemaConstants.SCHEMA_KIND);
KeyFactory keyFactory = datastore.newKeyFactory().setNamespace(SchemaConstants.NAMESPACE).setKind(SchemaConstants.SCHEMA_KIND);
Entity superSededEntity = datastore.get(keyFactory.newKey(entity.getString(SchemaConstants.SUPERSEDED_BY)));
superSededBy = getSchemaIdentity(superSededEntity);
}
......@@ -223,7 +223,7 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
entityBuilder.set(SchemaConstants.CREATED_BY, headers.getUserEmail());
entityBuilder.set(SchemaConstants.AUTHORITY, schema.getSchemaInfo().getSchemaIdentity().getAuthority());
entityBuilder.set(SchemaConstants.SOURCE, schema.getSchemaInfo().getSchemaIdentity().getSource());
entityBuilder.set(SchemaConstants.ENTITY_TYPE, schema.getSchemaInfo().getSchemaIdentity().getEntity());
entityBuilder.set(SchemaConstants.ENTITY_TYPE, schema.getSchemaInfo().getSchemaIdentity().getEntityType());
entityBuilder.set(SchemaConstants.MAJOR_VERSION,
schema.getSchemaInfo().getSchemaIdentity().getSchemaVersionMajor());
entityBuilder.set(SchemaConstants.MINOR_VERSION,
......@@ -243,7 +243,7 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
return SchemaIdentity.builder().id(entity.getKey().getName())
.authority(entity.getString(SchemaConstants.AUTHORITY)).source(entity.getString(SchemaConstants.SOURCE))
.entity(entity.getString(SchemaConstants.ENTITY_TYPE))
.entityType(entity.getString(SchemaConstants.ENTITY_TYPE))
.schemaVersionMajor(entity.getLong(SchemaConstants.MAJOR_VERSION))
.schemaVersionMinor(entity.getLong(SchemaConstants.MINOR_VERSION))
.schemaVersionPatch(entity.getLong(SchemaConstants.PATCH_VERSION)).build();
......@@ -306,8 +306,8 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
}
private boolean checkEntityMatch(SchemaInfo previousSchemaInfo, SchemaInfo schemaInfoObject) {
return schemaInfoObject.getSchemaIdentity().getEntity()
.equalsIgnoreCase(previousSchemaInfo.getSchemaIdentity().getEntity());
return schemaInfoObject.getSchemaIdentity().getEntityType()
.equalsIgnoreCase(previousSchemaInfo.getSchemaIdentity().getEntityType());
}
private boolean checkSourceMatch(SchemaInfo previousSchemaInfo, SchemaInfo schemaInfoObject) {
......@@ -328,8 +328,8 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
if (queryParams.getSource() != null) {
filterList.add(PropertyFilter.eq(SchemaConstants.SOURCE, queryParams.getSource()));
}
if (queryParams.getEntity() != null) {
filterList.add(PropertyFilter.eq(SchemaConstants.ENTITY_TYPE, queryParams.getEntity()));
if (queryParams.getEntityType() != null) {
filterList.add(PropertyFilter.eq(SchemaConstants.ENTITY_TYPE, queryParams.getEntityType()));
}
if (queryParams.getSchemaVersionMajor() != null) {
filterList.add(PropertyFilter.eq(SchemaConstants.MAJOR_VERSION, queryParams.getSchemaVersionMajor()));
......
package org.opengroup.osdu.schema.impl.sanityTestRunner;
import org.junit.Test;
import org.junit.internal.TextListener;
import org.junit.runner.JUnitCore;
import org.opengroup.osdu.schema.runner.TearDownTestsRunner;
public class TriggerPostIntegrationTestsRunner {
@Test
public void testError() {
JUnitCore junit = new JUnitCore();
junit.addListener(new TextListener(System.out));
junit.run(TearDownTestsRunner.class);
}
}
package org.opengroup.osdu.schema.impl.sanityTestRunner;
import org.junit.Test;
import org.junit.internal.TextListener;
import org.junit.runner.JUnitCore;
import org.opengroup.osdu.schema.runner.PreIntegrationTestsRunner;
public class TriggerPreIntegrationTestsRunner {
@Test
public void testError() {
JUnitCore junit = new JUnitCore();
junit.addListener(new TextListener(System.out));
junit.run(PreIntegrationTestsRunner.class);
}
}
package org.opengroup.osdu.schema.impl.sanityTestRunner;
import org.junit.Test;
import org.junit.internal.TextListener;
import org.junit.runner.JUnitCore;
import org.opengroup.osdu.schema.runner.SchemaServiceTestsRunner;
public class TriggerSchemaServiceTestsRunner {
@Test
public void testError() {
JUnitCore junit = new JUnitCore();
junit.addListener(new TextListener(System.out));
junit.run(SchemaServiceTestsRunner.class);
}
}
......@@ -134,7 +134,7 @@ public class GoogleEntityTypeStoreTest {
googleEntityStore.create(mockEntityType);
fail("Should not succeed");
} catch (BadRequestException e) {
assertEquals("Entity already registered with Id: wks", e.getMessage());
assertEquals("EntityType already registered with Id: wks", e.getMessage());
} catch (Exception e) {
fail("Should not get different exception");
......
......@@ -420,7 +420,7 @@ public class GoogleSchemaInfoStoreTest {
Mockito.when(queryResult.hasNext()).thenReturn(true, false);
Mockito.when(queryResult.next()).thenReturn(getMockEntityObject());
assertEquals(1,
schemaInfoStore.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entity("test")
schemaInfoStore.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test")
.schemaVersionMajor(1l).schemaVersionMinor(1l).scope("test").status("test").latestVersion(false)
.limit(100).offset(0).build(), "test").size());
}
......@@ -438,7 +438,7 @@ public class GoogleSchemaInfoStoreTest {
Mockito.when(queryResult.hasNext()).thenReturn(false);
assertEquals(0,
schemaInfoStore
.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entity("test")
.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test")
.scope("test").status("test").latestVersion(true).limit(100).offset(0).build(), "test")
.size());
}
......@@ -597,7 +597,7 @@ public class GoogleSchemaInfoStoreTest {
private SchemaRequest getMockSchemaObject_Published() {
return SchemaRequest.builder().schema("{}")
.schemaInfo(SchemaInfo.builder()
.schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entity("well")
.schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entityType("well")
.schemaVersionMajor(1L).schemaVersionMinor(1L).schemaVersionPatch(1L)
.id("os:wks:well.1.1.1").build())
.scope(SchemaScope.SHARED).status(SchemaStatus.PUBLISHED).createdBy("subham").build())
......@@ -606,7 +606,7 @@ public class GoogleSchemaInfoStoreTest {
}
private SchemaInfo getMockSchemaInfo() {
return SchemaInfo.builder().schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entity("well")
return SchemaInfo.builder().schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entityType("well")
.schemaVersionMajor(1L).schemaVersionMinor(1L).schemaVersionPatch(1L).id("os:wks:well.1.1.1").build())
.scope(SchemaScope.SHARED).status(SchemaStatus.PUBLISHED).createdBy("subham")
......@@ -617,11 +617,11 @@ public class GoogleSchemaInfoStoreTest {
private SchemaRequest getMockSchemaObject_SuperSededBy() {
return SchemaRequest.builder().schema("{}")
.schemaInfo(SchemaInfo.builder()
.schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entity("well")
.schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entityType("well")
.schemaVersionMajor(1L).schemaVersionMinor(1L).schemaVersionPatch(1L)
.id("os:wks:well.1.1.1").build())
.scope(SchemaScope.SHARED).status(SchemaStatus.PUBLISHED).createdBy("subham")
.supersededBy(SchemaIdentity.builder().authority("os").source("wks").entity("well")
.supersededBy(SchemaIdentity.builder().authority("os").source("wks").entityType("well")
.schemaVersionMajor(1L).schemaVersionMinor(1L).schemaVersionPatch(2L)
.id("os:wks:well.1.2.1").build())
.build())
......@@ -632,11 +632,11 @@ public class GoogleSchemaInfoStoreTest {
private SchemaRequest getMockSchemaObject_SuperSededByWithoutId() {
return SchemaRequest.builder().schema("{}")
.schemaInfo(SchemaInfo.builder()
.schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entity("well")
.schemaIdentity(SchemaIdentity.builder().authority("os").source("wks").entityType("well")
.schemaVersionMajor(1L).schemaVersionMinor(1L).schemaVersionPatch(1L)
.id("os:wks:well.1.1.1").build())
.scope(SchemaScope.SHARED).status(SchemaStatus.DEVELOPMENT).createdBy("subham")
.supersededBy(SchemaIdentity.builder().authority("os").source("wks").entity("well")
.supersededBy(SchemaIdentity.builder().authority("os").source("wks").entityType("well")
.schemaVersionMajor(1L).schemaVersionMinor(1L).schemaVersionPatch(1L).build())
.build())
.build();
......
......@@ -143,6 +143,11 @@
<version>1.76.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
</dependencies>
<repositories>
......@@ -185,14 +190,63 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.6</version>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M3</version>
<configuration>
<skipTests>${skipUnitTests}</skipTests>
<forkCount>3</forkCount>
<reuseForks>true</reuseForks>
<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.0.0-M3</version>
<executions>
<execution>
<id>Jar Tests Package</id>
<phase>package</phase>
<id>pre-integration-test</id>
<goals>
<goal>test-jar</goal>
<goal>integration-test</goal>
</goals>
<configuration>
<skipTests>${skipItTests}</skipTests>
<includes>
<include>org.opengroup.osdu.schema.runner.PreIntegrationTestsRunner</include>
</includes>
</configuration>
</execution>
<execution>
<id>integration-test</id>
<goals>
<goal>integration-test</goal>
</goals>
<configuration>
<skipTests>${skipItTests}</skipTests>
<excludes>
<exclude>none</exclude>
</excludes>
<includes>
<include>org.opengroup.osdu.schema.runner.SchemaServiceTestsRunner</include>
</includes>
</configuration>
</execution>
<execution>
<id>post-integration-test</id>
<goals>
<goal>integration-test</goal>
</goals>
<configuration>
<skipTests>${skipItTests}</skipTests>
<includes>
<include>org.opengroup.osdu.schema.runner.TearDownTestsRunner</include>
</includes>
</configuration>
</execution>
<execution>
<id>verify</id>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
......@@ -200,4 +254,31 @@
</plugins>
</build>
<profiles>
<profile>
<id>Default</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<skipUnitTests>false</skipUnitTests>
<skipItTests>true</skipItTests>
<preIntegrationPhase>none</preIntegrationPhase>
<postIntegrationPhase>none</postIntegrationPhase>
<mainClass>org.opengroup.osdu.schema.SchemaApplication</mainClass>
</properties>
</profile>
<profile>
<id>IntegrationTest</id>
<properties>
<skipUnitTests>true</skipUnitTests>
<skipItTests>false</skipItTests>
<preIntegrationPhase>pre-integration-test</preIntegrationPhase>
<postIntegrationPhase>post-integration-test</postIntegrationPhase>
<mainClass>org.opengroup.osdu.schema.SchemaApplication</mainClass>
</properties>
</profile>
</profiles>
</project>
\ No newline at end of file
package org.opengroup.osdu.schema.api;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class HealthCheckController {
@GetMapping("/health")
public String healthMessage() {
return "Alive";
}
}
......@@ -52,7 +52,7 @@ public class SchemaController {
public ResponseEntity<SchemaInfoResponse> getSchemaInfoList(
@RequestParam(required = false, name = "authority") String authority,
@RequestParam(required = false, name = "source") String source,
@RequestParam(required = false, name = "entityType") String entity,
@RequestParam(required = false, name = "entityType") String entityType,
@RequestParam(required = false, name = "schemaVersionMajor") Long schemaVersionMajor,
@RequestParam(required = false, name = "schemaVersionMinor") Long schemaVersionMinor,
@RequestParam(required = false, name = "status") String status,
......@@ -61,7 +61,7 @@ public class SchemaController {
@RequestParam(required = false, name = "limit", defaultValue = "100") int limit,
@RequestParam(required = false, name = "offset", defaultValue = "0") int offset)
throws ApplicationException, NotFoundException, BadRequestException {
QueryParams queryParams = QueryParams.builder().authority(authority).source(source).entity(entity)
QueryParams queryParams = QueryParams.builder().authority(authority).source(source).entityType(entityType)