Commit c79fee75 authored by Mark Hewitt's avatar Mark Hewitt
Browse files

merge dataload checkrefs code (wip)

parent 60e7cef4
Pipeline #71109 passed with stages
in 6 minutes and 17 seconds
......@@ -32,6 +32,7 @@ Change Log
------
- schema add --overwrite-existing option
- merge dataload checkrefs code (wip)
0.0.18
------
......
......@@ -12,4 +12,4 @@
""" OSDU command line environment"""
__VERSION__ = "0.0.18"
__VERSION__ = "0.0.19"
This diff is collapsed.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for OSDU CLI"""
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test cases for dataload checkrefs command"""
from knack.testsdk import ScenarioTest
class UnitTests(ScenarioTest):
"""Test cases for dataload checkrefs commands
Uses the VCR library to record / replay HTTP requests into a file.
"""
def __init__(self, method_name):
super().__init__(None, method_name, filter_headers=["Authorization"])
self.recording_processors = [self.name_replacer]
self.vcr.register_matcher("always", UnitTests._vcrpy_match_always)
self.vcr.match_on = ["always"]
@classmethod
def _vcrpy_match_always(cls, url1, url2): # pylint: disable=W0613
"""Return true always (only 1 query)."""
return True
if __name__ == "__main__":
import nose2
nose2.main()
......@@ -16,33 +16,18 @@
import logging
from knack.testsdk import ScenarioTest
from mock import MagicMock, patch
from mock import patch
from osdu.identity import OsduTokenCredential
from testfixtures import LogCapture
from osducli.click_cli import State
from osducli.cliclient import CliOsduClient
from osducli.commands.unit.list import unit_list
from osducli.config import CONFIG_AUTHENTICATION_MODE, CONFIG_SERVER
def mock_config_values(section, name, fallback=None): # pylint: disable=W0613
"""Validate and mock config returns"""
# if section != 'core':
# raise ValueError(f'Cannot retrieve config section \'{section}\'')
if name == CONFIG_SERVER:
return "https://dummy.com"
if name == CONFIG_AUTHENTICATION_MODE:
return "refresh_token"
return f"{section}_{name}"
MOCK_CONFIG = MagicMock()
MOCK_CONFIG.get.side_effect = mock_config_values
from tests.helpers import MockConfig
class UnitTests(ScenarioTest):
"""Test cases for unit commands
"""Test cases for unit list command
Uses the VCR library to record / replay HTTP requests into
a file.
......@@ -66,7 +51,7 @@ class UnitTests(ScenarioTest):
self.cassette.filter_headers = ["Authorization"]
state = State()
state.config = MOCK_CONFIG
state.config = MockConfig
with LogCapture(level=logging.WARN) as log_capture:
result = unit_list(state)
......@@ -80,7 +65,7 @@ class UnitTests(ScenarioTest):
def test_unit_list_exit(self, mock_cli_get_returning_json): # pylint: disable=W0613
"""Test any exit error is propogated"""
state = State()
state.config = MOCK_CONFIG
state.config = MockConfig
with self.assertRaises(SystemExit) as sysexit:
_ = unit_list(state)
......
......@@ -406,6 +406,7 @@ class HelpTextTests(unittest.TestCase):
self.validate_output(
"osdu dataload",
commands=(
"checkrefs",
"ingest",
"status",
"verify",
......
......@@ -12,102 +12,21 @@
"""Shared helpers for mocks and utils used among all tests"""
import os
import xml.etree.ElementTree as ET
from mock import MagicMock
from osducli.config import CONFIG_AUTHENTICATION_MODE, CONFIG_SERVER
def get_mock_endpoint():
"""Get the endpoint for a mock cluster."""
return os.environ.get("SF_TEST_ENDPOINT")
def set_mock_endpoint(endpoint):
"""Set the endpoint for a mock cluster.
param endpoint: string"""
if endpoint is not None:
os.environ["SF_TEST_ENDPOINT"] = endpoint
else:
try:
del os.environ["SF_TEST_ENDPOINT"]
except KeyError:
# Do nothing if the key doesn't exist
pass
APP_PATH = os.environ.get("SF_TEST_APP_PATH", False)
ENDPOINT = get_mock_endpoint()
MOCK_CONFIG = MagicMock()
def mock_config_values(section, name, fallback):
def mock_config_values(section, name, fallback=None): # pylint: disable=W0613
"""Validate and mock config returns"""
if section != "core":
raise ValueError("Cannot retrieve non core config value")
if name == "endpoint":
return get_mock_endpoint()
if name == "security":
return "none"
return fallback
MOCK_CONFIG.return_value.get.side_effect = mock_config_values
# XMLNS for fabric manifests
XML_NS = {"fabric": "http://schemas.microsoft.com/2011/01/fabric"}
def parse_app_version(xml_file):
"""Parse application type version from application manifest"""
root = ET.parse(xml_file).getroot()
version = root.attrib.get("ApplicationTypeVersion", None)
if not version:
raise ValueError("Could not parse application type version")
return version
def parse_app_type(xml_file):
"""Parse application type name from application manifest"""
root = ET.parse(xml_file).getroot()
app_type = root.attrib.get("ApplicationTypeName", None)
if not app_type:
raise ValueError("Could not parse application type name")
return app_type
def find_service_manifest(xml_file):
"""Find the path to the first service manifest for an application"""
root = ET.parse(xml_file).getroot()
import_elem = root.find("fabric:ServiceManifestImport", XML_NS)
if import_elem is None:
raise ValueError("Could not find service manifest import section")
ref_elem = import_elem.find("fabric:ServiceManifestRef", XML_NS)
if ref_elem is None:
raise ValueError("Could not find service manifest reference section")
manifest_name = ref_elem.attrib.get("ServiceManifestName", None)
if not manifest_name:
raise ValueError("Could not find service manifest name")
return os.path.join(os.path.dirname(xml_file), manifest_name, "ServiceManifest.xml")
# if section != 'core':
# raise ValueError(f'Cannot retrieve config section \'{section}\'')
if name == CONFIG_SERVER:
return "https://dummy.com"
if name == CONFIG_AUTHENTICATION_MODE:
return "refresh_token"
return f"{section}_{name}"
def parse_service_type(xml_file):
"""Determines the first available service type and the associated
type name"""
root = ET.parse(xml_file).getroot()
service_type = root.find("fabric:ServiceTypes", XML_NS)
if service_type is None:
raise ValueError("Could not find service types in service manifest")
service_type_kind = None
if "StatelessServiceType" in service_type[0].tag:
service_type_kind = "stateless"
else:
# For now we only support stateless services for service creation
raise ValueError("Unsupported service type")
service_type_name = service_type[0].attrib.get("ServiceTypeName", None)
if not service_type_name:
raise ValueError("Could not find service type name")
return service_type_kind, service_type_name
MockConfig = MagicMock()
MockConfig.get.side_effect = mock_config_values
......@@ -23,24 +23,14 @@ from requests.models import HTTPError
from testfixtures import LogCapture
from osducli.cliclient import MSG_HTTP_ERROR, CliOsduClient
from osducli.config import CONFIG_AUTHENTICATION_MODE, CONFIG_SERVER
from osducli.config import CONFIG_AUTHENTICATION_MODE
from tests.helpers import MockConfig, mock_config_values
SAMPLE_JSON = {
"name": "value",
}
def mock_config_values(section, name, fallback=None): # pylint: disable=W0613
"""Validate and mock config returns"""
# if section != 'core':
# raise ValueError(f'Cannot retrieve config section \'{section}\'')
if name == CONFIG_SERVER:
return "https://dummy.com"
if name == CONFIG_AUTHENTICATION_MODE:
return "refresh_token"
return f"{section}_{name}"
def mock_config_values_invalid_auth(section, name, fallback=None): # pylint: disable=W0613
"""Validate and mock config returns"""
if name == CONFIG_AUTHENTICATION_MODE:
......@@ -49,9 +39,6 @@ def mock_config_values_invalid_auth(section, name, fallback=None): # pylint: di
return mock_config_values(section, name, fallback)
MOCK_CONFIG = MagicMock()
MOCK_CONFIG.get.side_effect = mock_config_values
MOCK_CONFIG_INVALID_AUTH = MagicMock()
MOCK_CONFIG_INVALID_AUTH.get.side_effect = mock_config_values_invalid_auth
......@@ -79,7 +66,7 @@ class CliOsduClientTests(ScenarioTest):
def test_init(self):
"""Test the init method"""
client = CliOsduClient(MOCK_CONFIG)
client = CliOsduClient(MockConfig)
self.assertEqual(client.server_url, mock_config_values("core", "server"))
self.assertEqual(client.data_partition, mock_config_values("core", "data_partition_id"))
......@@ -112,7 +99,7 @@ class CliOsduClientTests(ScenarioTest):
self.cassette.filter_headers = ["Authorization"]
with LogCapture(level=logging.WARN) as log_capture:
connection = CliOsduClient(MOCK_CONFIG)
connection = CliOsduClient(MockConfig)
result = connection.cli_get_returning_json("unit_url", "unit?limit=3")
assert isinstance(result, dict)
assert result["count"] == 3
......@@ -129,7 +116,7 @@ class CliOsduClientTests(ScenarioTest):
"""Test 404 errors return the correct message"""
with self.assertRaises(HTTPError) as sysexit:
with LogCapture(level=logging.INFO) as log_capture:
connection = CliOsduClient(MOCK_CONFIG)
connection = CliOsduClient(MockConfig)
_ = connection.cli_get_returning_json("DUMMY_KEY", "DUMMY_STRING")
log_capture.check_present(("cli", "ERROR", MSG_HTTP_ERROR))
self.assertEqual(sysexit.exception.code, 1)
......@@ -168,7 +155,7 @@ class CliOsduClientTests(ScenarioTest):
"""Test valid put with string returns expected values"""
response_mock = Mock()
with patch.object(OsduClient, "put", return_value=response_mock) as mock_put:
client = CliOsduClient(MOCK_CONFIG)
client = CliOsduClient(MockConfig)
response = client.cli_put(config, path, data)
......@@ -180,7 +167,7 @@ class CliOsduClientTests(ScenarioTest):
"""Test valid put with string returns expected values"""
response_mock = Mock()
with patch.object(OsduClient, "put", return_value=response_mock) as mock_put:
client = CliOsduClient(MOCK_CONFIG)
client = CliOsduClient(MockConfig)
response = client.cli_put("config", "/path", "data")
......@@ -200,7 +187,7 @@ class CliOsduClientTests(ScenarioTest):
response_mock = Mock()
type(response_mock).status_code = PropertyMock(return_value=actual_status_code)
with patch.object(OsduClient, "put", return_value=response_mock) as mock_put:
client = CliOsduClient(MOCK_CONFIG)
client = CliOsduClient(MockConfig)
response = client.cli_put("config", "/path", SAMPLE_JSON, ok_status_codes)
......@@ -218,7 +205,7 @@ class CliOsduClientTests(ScenarioTest):
type(response_mock).status_code = PropertyMock(return_value=actual_status_code)
with patch.object(OsduClient, "put", return_value=response_mock) as _:
with self.assertRaises(HTTPError):
client = CliOsduClient(MOCK_CONFIG)
client = CliOsduClient(MockConfig)
client.cli_put("config", "/path", SAMPLE_JSON, ok_status_codes)
......@@ -237,7 +224,7 @@ class CliOsduClientTests(ScenarioTest):
"""Test valid put with string returns expected values"""
response_mock = Mock()
with patch.object(OsduClient, "put_returning_json", return_value=response_mock) as mock_put:
client = CliOsduClient(MOCK_CONFIG)
client = CliOsduClient(MockConfig)
response = client.cli_put_returning_json(config, path, string_data, status_codes)
......@@ -251,7 +238,7 @@ class CliOsduClientTests(ScenarioTest):
"""Test valid put with string returns expected values"""
response_mock = Mock()
with patch.object(OsduClient, "put_returning_json", return_value=response_mock) as mock_put:
client = CliOsduClient(MOCK_CONFIG)
client = CliOsduClient(MockConfig)
response = client.cli_put_returning_json("config", "/path", "data")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment