test_blob_storage_gcp.py 5.81 KB
Newer Older
1
2
import tempfile
import shutil
Yannick's avatar
Yannick committed
3
4
import uuid

5
6
import pytest
import aiohttp
Yannick's avatar
Yannick committed
7
8
9
from tests.conftest import *

from osdu_gcp.storage.blob_storage_gcp import GCloudAioStorage
Luc Yriarte's avatar
Luc Yriarte committed
10
from osdu.core.api.storage.tenant import Tenant
Yannick's avatar
Yannick committed
11
from osdu.core.api.storage.exceptions import *
12

Yannick's avatar
Yannick committed
13

14
15
16
class _TESTING_CFG:
    credentials = TESTING_GCP_DATA_PROJECT_CREDENTIALS

Yannick's avatar
Yannick committed
17

18
19
20
21
22
23
24
25
26
TEST_DATA = {
    'initial_files': [
        # object_name , content
        ('test_file_1.txt', 'content of test file 1'),
        ('test_file_2.txt', 'content of test file 2')
    ]
}


27
28
29
30
31
32
33
34
35
36
37
38
# test local and gcp storage
@pytest.fixture(params=['GCloudAioStorage'])
async def storage_client(request):
    client_name = request.param
    if client_name == 'GCloudAioStorage':
        session = aiohttp.ClientSession()
        yield GCloudAioStorage(session=session, service_account_file=_TESTING_CFG.credentials)
        await session.close()


@pytest.fixture
async def test_tenant():
39
    return Tenant(project_id=TESTING_GCP_DATA_PROJECT_ID, bucket_name=TESTING_GCP_BUCKET,
Yannick's avatar
Yannick committed
40
41
                  data_partition_id='testing-partition-name')

42
43
44
45
46
47
48
49
50

@pytest.fixture
async def temp_directory() -> str:
    tmpdir = tempfile.mkdtemp()
    yield tmpdir

    # teardown - recursively delete the tmp directory
    shutil.rmtree(tmpdir, ignore_errors=True)

Yannick's avatar
Yannick committed
51

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
@pytest.mark.asyncio
async def test_list_objects(storage_client, test_tenant):
    result = await storage_client.list_objects(test_tenant)
    for file_name, _ in TEST_DATA['initial_files']:
        assert file_name in result


@pytest.mark.asyncio
async def test_download(storage_client, test_tenant):
    name, expected_content = TEST_DATA['initial_files'][0]
    data = await storage_client.download(test_tenant, name)
    result = data.decode('utf-8')
    assert result == expected_content


@pytest.mark.asyncio
async def test_upload_check_delete(storage_client, test_tenant):
    name = str(uuid.uuid4())
    content = str(uuid.uuid4())

    # upload
    await storage_client.upload(test_tenant, name, content, content_type='text/plain')

    # check single object with this name
    result = await storage_client.list_objects(test_tenant, prefix=name)
    assert result == [name]

    # check its content
    data = await storage_client.download(test_tenant, name)
    assert data.decode('utf-8') == content

    # delete it
    await storage_client.delete(test_tenant, name)

    # check nothing remains
    result = await storage_client.list_objects(test_tenant, prefix=name)
    assert len(result) == 0


@pytest.mark.asyncio
async def test_upload_file_bin_input(storage_client, temp_directory, test_tenant):
    file_c = temp_directory + '\\testing.file'
    with open(file_c, 'w') as f:
        f.write('expected content 123456789')
    content_bin = b'expected content 123456789'
    with open(file_c, 'rb') as file_bin_input:
        await storage_client.upload(test_tenant, 'file_bin_input', file_bin_input)
Yannick's avatar
Yannick committed
99
100
    assert await storage_client.download(test_tenant, 'file_bin_input') == content_bin

101
102
103
104
105
106
107
108
109

@pytest.mark.asyncio
async def test_upload_file_txt_input(storage_client, temp_directory, test_tenant):
    file_c = temp_directory + '\\testing.file'
    with open(file_c, 'w') as f:
        f.write('expected content 123456789')
    content_bin = b'expected content 123456789'
    with open(file_c, 'r') as file_txt_input:
        await storage_client.upload(test_tenant, 'file_txt_input', file_txt_input)
Yannick's avatar
Yannick committed
110
111
    assert await storage_client.download(test_tenant, 'file_txt_input') == content_bin

112
113
114
115
116
117

@pytest.mark.asyncio
async def test_upload_str_input(storage_client, test_tenant):
    content_bin = b'expected content 123456789'
    content_str = content_bin.decode('utf-8')
    await storage_client.upload(test_tenant, 'str_input', content_str)
Yannick's avatar
Yannick committed
118
119
    assert await storage_client.download(test_tenant, 'str_input') == content_bin

120
121
122
123
124

@pytest.mark.asyncio
async def test_upload_bin_input(storage_client, test_tenant):
    content_bin = b'expected content 123456789'
    await storage_client.upload(test_tenant, 'bin_input', content_bin)
Yannick's avatar
Yannick committed
125
126
    assert await storage_client.download(test_tenant, 'bin_input') == content_bin

127
128
129
130
131
132
133

@pytest.mark.asyncio
async def test_upload_empty_input(storage_client, test_tenant):
    await storage_client.upload(test_tenant, 'empty_input', None)
    actual_data = await storage_client.download(test_tenant, 'empty_input')
    assert len(actual_data) == 0

Yannick's avatar
Yannick committed
134

135
136
@pytest.mark.asyncio
async def test_upload_int_input(storage_client, test_tenant):
137
    with pytest.raises(TypeError):
138
139
140
        await storage_client.upload(test_tenant, 'int_input', 123456)


Yannick's avatar
Yannick committed
141
142
143
144
145
@pytest.mark.asyncio
async def test_overwrite_with_condition(storage_client, test_tenant):
    blob_name = 'testing_data/' + str(uuid.uuid4())
    await storage_client.upload(test_tenant, blob_name, b'1111')

Yannick's avatar
Yannick committed
146
    with pytest.raises(ResourceExistsException):
Yannick's avatar
Yannick committed
147
        await storage_client.upload(test_tenant, blob_name, b'1111', overwrite=False)
148

Yannick's avatar
Yannick committed
149
150
151
    # update no condition
    await storage_client.upload(test_tenant, blob_name, b'1112')
    assert await storage_client.download(test_tenant, blob_name) == b'1112'
152

Yannick's avatar
Yannick committed
153
154
155
156
    # successful update if_match
    etag_1112 = (await storage_client.download_metadata(test_tenant, blob_name)).etag
    await storage_client.upload(test_tenant, blob_name, b'1113', if_match=etag_1112)
    assert await storage_client.download(test_tenant, blob_name) == b'1113'
157

Yannick's avatar
Yannick committed
158
    # should fail update if_match not satisfied
Yannick's avatar
Yannick committed
159
    with pytest.raises(PreconditionFailedException):
Yannick's avatar
Yannick committed
160
        await storage_client.upload(test_tenant, blob_name, b'1114', if_match=etag_1112)
161

Yannick's avatar
Yannick committed
162
163
    # success update if_not_match
    await storage_client.upload(test_tenant, blob_name, b'1115', if_not_match=etag_1112)
164

Yannick's avatar
Yannick committed
165
166
    # should fail update if_not_match not satisfied
    etag_1115 = (await storage_client.download_metadata(test_tenant, blob_name)).etag
Yannick's avatar
Yannick committed
167
168
    with pytest.raises(PreconditionFailedException):
        await storage_client.upload(test_tenant, blob_name, b'1116', if_not_match=etag_1115)