Commit ed4edfb0 authored by Dmitry Kniazev's avatar Dmitry Kniazev
Browse files

Merge branch 'definition/create_stream' into 'develop'

Continue implementing the stream registration

See merge request !4
parents cf3a443d 2be21151
Pipeline #69405 failed with stages
in 3 minutes and 14 seconds
...@@ -30,4 +30,5 @@ application-local.properties ...@@ -30,4 +30,5 @@ application-local.properties
.vscode .vscode
### Python environment ### Python environment
.venv .venv
\ No newline at end of file __pycache__
\ No newline at end of file
...@@ -48,7 +48,7 @@ To build the executable jar file. ...@@ -48,7 +48,7 @@ To build the executable jar file.
``` bash ``` bash
mvn clean package mvn clean package
``` ```
Executing the above command generates the controllers interfaces based on the streaming api spec(open api format). The spec is part of the doc folder under the source tree. Executing the above command generates the controllers interfaces based on the streaming api spec(open api format). The spec is part of the doc folder under the source tree. The generation process is controlled by the maven plugin and is configured in the pom.xml (see [configuration options reference](https://openapi-generator.tech/docs/generators/spring))
To run the executable as a spring boot application To run the executable as a spring boot application
......
openapi: 3.0.0 openapi: 3.0.0
info: info:
version: 1.0.0 version: 1.0.0
title: OSDU Streaming Service title: OSDU Streaming Services
license: license:
name: Apache 2.0 name: Apache 2.0
url: "http://www.apache.org/licenses/LICENSE-2.0" url: "http://www.apache.org/licenses/LICENSE-2.0"
servers: servers:
- url: "https://localhost:8080/api/streaming/v1" - url: "http://localhost:8080/api/streaming/v1"
description: "Local development server"
tags: tags:
- name: stream-admin-api - name: stream-admin-api
description: Stream Admin API description: Stream Admin API
...@@ -62,7 +63,7 @@ paths: ...@@ -62,7 +63,7 @@ paths:
content: content:
application/json: application/json:
schema: schema:
$ref: "#/components/schemas/StreamDataset" $ref: "#/components/schemas/StreamRecord"
responses: responses:
"201": "201":
description: Created description: Created
...@@ -311,6 +312,8 @@ components: ...@@ -311,6 +312,8 @@ components:
$ref: "#/components/schemas/Acl" $ref: "#/components/schemas/Acl"
legal: legal:
$ref: "#/components/schemas/Legal" $ref: "#/components/schemas/Legal"
tags:
$ref: "#/components/schemas/Tags"
meta: meta:
type: array type: array
items: items:
...@@ -341,10 +344,12 @@ components: ...@@ -341,10 +344,12 @@ components:
- source - source
- processor - processor
- sink - sink
description: 'Type of the stream. Enumeration: "SOURCE", "PROCESSOR", "SINK"' description: 'Type of the stream. Enumeration: "source", "processor", "sink"'
example: "processor" example: "processor"
StreamDefinition: StreamDefinition:
$ref: "#/components/schemas/StreamDefinition" $ref: "#/components/schemas/StreamDefinition"
ExtensionProperties:
type: object
StreamDefinition: StreamDefinition:
type: object type: object
required: required:
...@@ -360,7 +365,7 @@ components: ...@@ -360,7 +365,7 @@ components:
items: items:
type: string type: string
example: "opendes_4dc4e8ec354e4953b6968fcb1d9d9f38_work-product-component--WellLog_1.0.0" example: "opendes_4dc4e8ec354e4953b6968fcb1d9d9f38_work-product-component--WellLog_1.0.0"
SubscribeIDs: SubscribeIds:
type: array type: array
items: items:
type: string type: string
...@@ -424,6 +429,10 @@ components: ...@@ -424,6 +429,10 @@ components:
enum: enum:
- compliant - compliant
- incompliant - incompliant
Tags:
type: object
additionalProperties:
type: object
Map: Map:
type: object type: object
additionalProperties: additionalProperties:
......
...@@ -53,6 +53,11 @@ ...@@ -53,6 +53,11 @@
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId> <artifactId>spring-boot-starter-test</artifactId>
...@@ -132,6 +137,7 @@ ...@@ -132,6 +137,7 @@
<configOptions> <configOptions>
<delegatePattern>false</delegatePattern> <delegatePattern>false</delegatePattern>
<interfaceOnly>true</interfaceOnly> <interfaceOnly>true</interfaceOnly>
<useBeanValidation>false</useBeanValidation>
</configOptions> </configOptions>
</configuration> </configuration>
</execution> </execution>
......
Jinja2==3.0.1
MarkupSafe==2.0.1
pkg-resources==0.0.0
python-dotenv==0.19.0 python-dotenv==0.19.0
from http import HTTPStatus
import http.client import http.client
import json import json
import argparse import argparse
import os import os
from dotenv import load_dotenv from dotenv import load_dotenv
from jinja2 import Environment, FileSystemLoader, select_autoescape
env = Environment(
loader=FileSystemLoader(searchpath="./templates"),
autoescape=select_autoescape())
def refresh_token(): def refresh_token():
...@@ -43,10 +48,34 @@ def get_stream(stream_id): ...@@ -43,10 +48,34 @@ def get_stream(stream_id):
conn.request( conn.request(
"GET", f"/api/streaming/v1/stream/{stream_id}", payload, headers) "GET", f"/api/streaming/v1/stream/{stream_id}", payload, headers)
res = conn.getresponse() res = conn.getresponse()
jsonString = res.read() jsonString = res.read().decode()
print(jsonString) print(jsonString)
def create_new_stream(stream_id, template_name):
# prepare the request body from template
template = env.get_template(f"{template_name}.json")
# prepare request headers
token = os.getenv('OSDU_API_TOKEN')
auth = f"Bearer {token}"
payload = template.render(record_id=stream_id)
headers = {'Content-Type': 'application/json',
'data-partition-id': 'opendes', 'Authorization': auth}
# make POST request to create a new stream
conn = http.client.HTTPConnection("localhost:8080")
conn.request(
"POST", f"/api/streaming/v1/stream", payload, headers)
res = conn.getresponse()
jsonString = res.read().decode()
if res.status == HTTPStatus.CREATED:
print("Stream successfully created - see full response below:")
print(jsonString)
else:
print(
f"Received the following status from server: {res.status}. See full response below:")
print(jsonString)
load_dotenv() load_dotenv()
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='Little helpers to use for Streaming API testing. They can help you with things like refreshing access tokens, posting a stream record, getting a stream record, etc.') description='Little helpers to use for Streaming API testing. They can help you with things like refreshing access tokens, posting a stream record, getting a stream record, etc.')
...@@ -56,11 +85,14 @@ parser.add_argument('command', type=str, choices=['refresh', 'get', 'post'], ...@@ -56,11 +85,14 @@ parser.add_argument('command', type=str, choices=['refresh', 'get', 'post'],
help='command to execute') help='command to execute')
parser.add_argument('--record-id', type=str, help='id of the stream record, e.g. opendes:dataset--Stream.Generic:12345', parser.add_argument('--record-id', type=str, help='id of the stream record, e.g. opendes:dataset--Stream.Generic:12345',
default='opendes:dataset--Stream.Generic:12345') default='opendes:dataset--Stream.Generic:12345')
parser.add_argument('--template', type=str, help='template for the stream record, e.g. source, processor, sink or your custom',
default='processor')
# main program here
args = parser.parse_args() args = parser.parse_args()
print( print(
f"Running helper [{args.helper}] with the following command [{args.command}]") f"Running helper [{args.helper}] with the following command [{args.command}]")
print(f"{args}") print(f"{args._get_args}")
if args.helper == "token": if args.helper == "token":
if args.command == "refresh": if args.command == "refresh":
...@@ -69,10 +101,9 @@ if args.helper == "token": ...@@ -69,10 +101,9 @@ if args.helper == "token":
print(f"No such command for helper [{args.helper}]") print(f"No such command for helper [{args.helper}]")
elif args.helper == "stream": elif args.helper == "stream":
if args.command == "get": if args.command == "get":
if args.record_id: get_stream(args.record_id)
get_stream(args.record_id) elif args.command == "post":
else: create_new_stream(args.record_id, args.template)
print(f"RECORD_ID cannot be empty with for the [{args.helper}]")
else: else:
print(f"No such command for helper [{args.helper}]") print(f"No such command for helper [{args.helper}]")
else: else:
......
{
"id": "{{ record_id }}",
"kind": "opendes:wks:dataset--Stream.Generic:1.0.0",
"acl": {
"owners": [
"data.default.owners@opendes.contoso.com"
],
"viewers": [
"data.default.viewers@opendes.contoso.com"
]
},
"legal": {
"legaltags": [
"opendes-public-usa-dataset-epam"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
},
"tags": {
"env": "dev-azure",
"auto-cleanup": "yes"
},
"ancestry": {
"parents": []
},
"meta": [],
"data": {
"ResourceSecurityClassification": "opendes:reference-data--ResourceSecurityClassification:RESTRICTED:",
"Source": "Contoso Inc.",
"Name": "Test processor stream",
"Description": "This stream is simulating the Kafka App processor that filters data and routes to myapp topic",
"DatasetProperties": {
"StreamType": "processor",
"StreamDefinition": {
"SubscribeIDs": [
"opendes:work-product-component--WellLog:be54a691c0384182944d71c6b2b6f699"
],
"SourceBindings": [
"opendes_wks_work-product-component--WellLog_1.0.0"
],
"SinkBindings": [
"opendes_myapp_work-product-component--WellLog_1.0.0"
]
}
}
}
}
\ No newline at end of file
{
"id": "{{ record_id }}",
"kind": "opendes:wks:dataset--Stream.Generic:1.0.0",
"acl": {
"owners": [
"data.default.owners@opendes.contoso.com"
],
"viewers": [
"data.default.viewers@opendes.contoso.com"
]
},
"legal": {
"legaltags": [
"opendes-public-usa-dataset-epam"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
},
"tags": {
"env": "dev-azure",
"auto-cleanup": "yes"
},
"ancestry": {
"parents": []
},
"meta": [],
"data": {
"ResourceSecurityClassification": "opendes:reference-data--ResourceSecurityClassification:RESTRICTED:",
"Source": "Contoso Inc.",
"Name": "Test sink stream",
"Description": "This stream is simulating the producer application that streams data into OSDU platform",
"DatasetProperties": {
"StreamType": "source",
"StreamDefinition": {
"SubscribeIDs": [
"opendes:work-product-component--WellLog:be54a691c0384182944d71c6b2b6f699"
],
"SourceBindings": [
"wss://remote-etp.server.com"
],
"SinkBindings": [
"opendes_wks_work-product-component--WellLog_1.0.0"
]
}
}
}
}
\ No newline at end of file
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
package org.opengroup.osdu.streaming.api; package org.opengroup.osdu.streaming.api;
import org.opengroup.osdu.streaming.StreamApi; import org.opengroup.osdu.streaming.StreamApi;
import org.opengroup.osdu.streaming.model.StreamDataset;
import org.opengroup.osdu.streaming.model.StreamRecord; import org.opengroup.osdu.streaming.model.StreamRecord;
import org.opengroup.osdu.streaming.service.StreamingAdminService; import org.opengroup.osdu.streaming.service.StreamingAdminService;
import org.slf4j.Logger; import org.slf4j.Logger;
...@@ -48,10 +47,10 @@ public class StreamingAdminControllerImpl implements StreamApi { ...@@ -48,10 +47,10 @@ public class StreamingAdminControllerImpl implements StreamApi {
private StreamingAdminService streamingAdminService; private StreamingAdminService streamingAdminService;
@Override @Override
public ResponseEntity<String> createNewStream(String dataPartitionId, StreamDataset streamDataset) { public ResponseEntity<String> createNewStream(String dataPartitionId, StreamRecord streamRecord) {
logger.debug("Creating a new stream: " + streamDataset.getName()); logger.debug("Creating a new stream: " + streamRecord.getData().getName());
String id = null; String id = null;
id = this.streamingAdminService.createNewStream(streamDataset); id = this.streamingAdminService.createNewStream(streamRecord);
if (id != null) { if (id != null) {
return new ResponseEntity<String>(id, HttpStatus.CREATED); return new ResponseEntity<String>(id, HttpStatus.CREATED);
} else { } else {
......
...@@ -14,13 +14,12 @@ ...@@ -14,13 +14,12 @@
package org.opengroup.osdu.streaming.service; package org.opengroup.osdu.streaming.service;
import org.opengroup.osdu.streaming.model.StreamDataset;
import org.opengroup.osdu.streaming.model.StreamRecord; import org.opengroup.osdu.streaming.model.StreamRecord;
public interface StreamingAdminService { public interface StreamingAdminService {
StreamRecord getStream(String streamRecordId); StreamRecord getStream(String streamRecordId);
String createNewStream(StreamDataset streamDataset); String createNewStream(StreamRecord streamRecord);
} }
...@@ -24,7 +24,6 @@ import org.opengroup.osdu.core.common.model.storage.StorageException; ...@@ -24,7 +24,6 @@ import org.opengroup.osdu.core.common.model.storage.StorageException;
import org.opengroup.osdu.core.common.model.storage.UpsertRecords; import org.opengroup.osdu.core.common.model.storage.UpsertRecords;
import org.opengroup.osdu.core.common.storage.IStorageFactory; import org.opengroup.osdu.core.common.storage.IStorageFactory;
import org.opengroup.osdu.core.common.storage.IStorageService; import org.opengroup.osdu.core.common.storage.IStorageService;
import org.opengroup.osdu.streaming.model.StreamDataset;
import org.opengroup.osdu.streaming.model.StreamRecord; import org.opengroup.osdu.streaming.model.StreamRecord;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -90,22 +89,30 @@ public class StreamingAdminServiceImpl implements StreamingAdminService { ...@@ -90,22 +89,30 @@ public class StreamingAdminServiceImpl implements StreamingAdminService {
// easier to expect all of these to be provided by the client, so that we can // easier to expect all of these to be provided by the client, so that we can
// pass it to the storage svc without any changes... // pass it to the storage svc without any changes...
@Override @Override
public String createNewStream(StreamDataset streamDataset) { public String createNewStream(StreamRecord streamRecord) {
UpsertRecords upRec = null; UpsertRecords upRec = null;
try { try {
// get the instance of StorageService
logger.debug("Creating a storage service with headers: " + headers.getHeaders().toString()); logger.debug("Creating a storage service with headers: " + headers.getHeaders().toString());
IStorageService storageService = this.storageFactory.create(headers); IStorageService storageService = this.storageFactory.create(headers);
// convert StreamRecord to Record
Record rec = new Record(); Record rec = new Record();
rec.setId("opendes:dataset--Stream.Kafka:12345"); String jsonRepr = new String();
rec.setKind("opendes:wks:dataset--Stream.Kafka:1.0.0"); jsonRepr = objectMapper.writeValueAsString(streamRecord);
logger.debug("Attempt to create a new record with:\nRecordId: " + rec.getId() + "\nRecordKind: " logger.debug("Extracted record: \n" + jsonRepr);
+ rec.getKind()); rec = objectMapper.readValue(jsonRepr, Record.class);
// try inserting a new record to storage
upRec = storageService.upsertRecord(rec); upRec = storageService.upsertRecord(rec);
} catch (StorageException e) { } catch (StorageException e) {
logger.error("Got exception: " + e.getMessage() + "\nFull HTTP Response:" + e.getHttpResponse()); logger.error("Got exception: " + e.getMessage() + "\nFull HTTP Response:" + e.getHttpResponse());
} catch (JsonProcessingException e) {
logger.error("Got exception: " + e.getMessage() + "\nLocation" + e.getLocation().toString());
} }
if (upRec != null && upRec.getRecordCount() > 0) { if (upRec != null && upRec.getRecordCount() > 0) {
return upRec.getRecordIds().get(0); return upRec.getRecordIds().get(0);
} else } else
......
package com.example.demo;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.boot.test.web.client.TestRestTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.junit4.SpringRunner;
import static org.assertj.core.api.Assertions.assertThat;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
public class DemoApplicationTests {
@Test
public void contextLoads() {
}
@Autowired
private TestRestTemplate restTemplate;
@Test
public void homeResponse() {
String body = this.restTemplate.getForObject("/", String.class);
assertThat(body).isEqualTo("Spring is here!");
}
}
package org.opengroup.osdu.streaming.api;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.test.web.reactive.server.WebTestClient;
@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
public class StreamingAdminControllerTest {
final String uri = "/stream/";
final String stream_id = "opendes:dataset--Stream.Generic:12345";
// TODO add support for requests with teh valid tokens
@Autowired
private WebTestClient webTestClient;
// should fail since no required headers
@Test
void getStreamById_expectHttp400() {
webTestClient.get().uri(uri + stream_id).exchange().expectStatus().isBadRequest();
}
// should fail since no JWT token
@Test
void getStreamById_expectHttp401() {
webTestClient.get().uri(uri + stream_id).header("data-partition-id", "opendes").exchange().expectStatus()
.isUnauthorized();
}
// should fail since JWT token is invalid
@Test
void getStreamById_expectHttp403() {
webTestClient.get().uri(uri + stream_id).header("data-partition-id", "opendes")
.header("Authorization", "Bearer eyXYZ").exchange().expectStatus().isForbidden();
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment