From d1bca15067dc9dae9c7f54946e82bf0ea0d6ad86 Mon Sep 17 00:00:00 2001
From: Dmitrii <dmitrii_valuiskii@epam.com>
Date: Wed, 3 Feb 2021 16:15:31 +0300
Subject: [PATCH] GONRG-1726: remove ids autogeneration
---
src/dags/libs/process_manifest_r3.py | 21 +++---------
.../test_manifest_processor_r3.py | 34 -------------------
2 files changed, 4 insertions(+), 51 deletions(-)
diff --git a/src/dags/libs/process_manifest_r3.py b/src/dags/libs/process_manifest_r3.py
index 3c2d1bb..804e5ec 100644
--- a/src/dags/libs/process_manifest_r3.py
+++ b/src/dags/libs/process_manifest_r3.py
@@ -49,7 +49,6 @@ class ManifestProcessor(HeadersMixin):
"legal": {},
"acl": {},
"kind": "",
- "id": "",
"data": {}
}
@@ -107,20 +106,6 @@ class ManifestProcessor(HeadersMixin):
logger.error(f"Unhandled exception while uploading {file_path}: {e}")
return file_record
- def generate_id(self, manifest_fragment: dict) -> str:
- """Generate id to use it in Storage.
-
- :param manifest_fragment: The manifest to update
- :type manifest_fragment: dict
- :return: The generated id
- :rtype: str
- """
- group_type = manifest_fragment.get("groupType", "doc")
- kind = manifest_fragment.get("kind")
- kind_name = self._get_kind_name(kind)
- _id = f"{self.context.data_partition_id}:{group_type}_{kind_name}:{str(uuid.uuid4())}"
- return _id
-
def populate_manifest_storage_record(self, manifest: dict) -> dict:
"""Create a record from manifest to store it in Storage service.
@@ -130,7 +115,8 @@ class ManifestProcessor(HeadersMixin):
:rtype: dict
"""
record = copy.deepcopy(self.RECORD_TEMPLATE)
- record["id"] = manifest["id"] if manifest.get("id") else self.generate_id(manifest)
+ if manifest.get("id"):
+ record["id"] = manifest["id"]
record["kind"] = manifest.pop("kind")
record["legal"] = manifest.pop("legal")
record["acl"] = manifest.pop("acl")
@@ -146,7 +132,8 @@ class ManifestProcessor(HeadersMixin):
:rtype: dict
"""
record = copy.deepcopy(self.RECORD_TEMPLATE)
- record["id"] = manifest["id"] if manifest.get("id") else self.generate_id(manifest)
+ if manifest.get("id"):
+ record["id"] = manifest["id"]
record["kind"] = manifest.pop("kind")
record["legal"] = manifest.pop("legal")
record["acl"] = manifest.pop("acl")
diff --git a/tests/plugin-unit-tests/test_manifest_processor_r3.py b/tests/plugin-unit-tests/test_manifest_processor_r3.py
index cce91b0..e3bd248 100644
--- a/tests/plugin-unit-tests/test_manifest_processor_r3.py
+++ b/tests/plugin-unit-tests/test_manifest_processor_r3.py
@@ -97,7 +97,6 @@ class TestManifestProcessor:
source_file_checker=source_file_checker,
)
- monkeypatch.setattr(manifest_processor, "generate_id", lambda manifest: "test_id")
monkeypatch.setattr(file_handler, "upload_file",
lambda *args, **kwargs: "/test/source_file")
monkeypatch.setattr(file_handler, "save_file_record",
@@ -233,36 +232,3 @@ class TestManifestProcessor:
for manifest_part in manifest_processor.manifest_records:
kind = manifest_part["entity"]["kind"]
assert expected_kind_name == manifest_processor._get_kind_name(kind)
-
- @pytest.mark.parametrize(
- "conf_path,expected_kind_name,traversal_manifest_file",
- [
- pytest.param(MANIFEST_WELLBORE_VALID_PATH, "Wellbore", TRAVERSAL_WELLBORE_VALID_PATH, id="Valid Wellbore"),
- ]
- )
- def test_generate_id(
- self,
- monkeypatch,
- conf_path: str,
- traversal_manifest_file: str,
- expected_kind_name: str
- ):
- with open(conf_path) as f:
- conf = json.load(f)
- with open(traversal_manifest_file) as f:
- manifest_file = json.load(f)
- manifest_records = manifest_file
- context = process_manifest_r3.Context.populate(conf)
- manifest_processor = process_manifest_r3.ManifestProcessor(
- storage_url="",
- manifest_records=manifest_records,
- token_refresher=AirflowTokenRefresher(),
- context=context,
- file_handler=FileHandler("test", AirflowTokenRefresher(), context),
- source_file_checker=SourceFileChecker()
- )
- for manifest_part in manifest_processor.manifest_records:
- group_type = manifest_part["entity"]["groupType"]
- kind_name = manifest_processor._get_kind_name(manifest_part["entity"]["kind"])
- generated_id = manifest_processor.generate_id(manifest_part["entity"])
- assert generated_id.startswith(f"{TENANT}:{group_type}_{kind_name}:")
--
GitLab