Skip to content
GitLab
Commit 62a9c102 authored by Sid Stenersen's avatar Sid Stenersen
Browse files

Merge branch 'slb/ss/ls' into 'master' 

Fix /utility/ls pagination bug

See merge request !44
parents 8f1b056e 8c63d513
Pipeline #34727 passed with stages
in 9 minutes and 57 seconds
Hide whitespace changes
Inline Side-by-side
src/services/dataset/dao.ts
View file @ 62a9c102
// ============================================================================
// Copyright 2017-2019, Schlumberger
// Copyright 2017-2021, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -122,27 +122,44 @@ export class DatasetDAO {
}
public static async paginatedListContent(
journalClient: IJournal | IJournalTransaction, dataset: DatasetModel, pagination: PaginationModel):
journalClient: IJournal | IJournalTransaction, dataset: DatasetModel, wmode: string, pagination: PaginationModel):
Promise<{ datasets: string[], nextPageCursor: string }> {
const output = { datasets: [], nextPageCursor: null };
// Retrieve the content datasets
let query = journalClient.createQuery(
Config.SEISMIC_STORE_NS + '-' + dataset.tenant + '-' + dataset.subproject, Config.DATASETS_KIND)
.filter('path', dataset.path);
if (pagination.cursor) {
query = query.start(pagination.cursor);
}
if (pagination.limit) {
query = query.limit(pagination.limit);
if (wmode !== Config.LS_MODE.DATASETS && !pagination.cursor) {
// Retrieve directories
const query = journalClient.createQuery(
Config.SEISMIC_STORE_NS + '-' + dataset.tenant + '-' + dataset.subproject, Config.DATASETS_KIND)
.select(['path']).groupBy('path').filter('path', '>', dataset.path).filter('path', '<', dataset.path + '\ufffd');
const [entitieshy] = await journalClient.runQuery(query);
output.datasets = entitieshy.map((entity) => ((entity.path || '') as string).substr(dataset.path.length));
output.datasets = output.datasets.map(
(entity) => entity.substr(0, entity.indexOf('/') + 1)).filter(
(elem, index, self) => index === self.indexOf(elem));
}
const [entitiesds, info] = await journalClient.runQuery(query);
if (entitiesds.length !== 0) {
output.datasets = entitiesds.map((item) => item.name);
if (pagination) {
output.nextPageCursor = info.endCursor;
if (wmode !== Config.LS_MODE.DIRS) {
// Retrieve datasets
let query = journalClient.createQuery(
Config.SEISMIC_STORE_NS + '-' + dataset.tenant + '-' + dataset.subproject, Config.DATASETS_KIND)
.filter('path', dataset.path);
if (pagination.cursor) {
query = query.start(pagination.cursor);
}
if (pagination.limit) {
query = query.limit(pagination.limit);
}
const [entitiesds, info] = await journalClient.runQuery(query);
if (entitiesds.length !== 0) {
output.datasets = output.datasets.concat(entitiesds.map((item) => item.name));
if (pagination) {
output.nextPageCursor = info.endCursor;
}
}
}
return output;
......@@ -183,7 +200,7 @@ export class DatasetDAO {
const results = { datasets: [], directories: [] };
// Retrieve the content datasets
if (wmode === Config.LS_MODE.ALL || wmode === Config.LS_MODE.DATASETS) {
if (wmode !== Config.LS_MODE.DIRS) {
const query = journalClient.createQuery(
Config.SEISMIC_STORE_NS + '-' + dataset.tenant + '-' + dataset.subproject, Config.DATASETS_KIND)
.filter('path', dataset.path);
......@@ -194,7 +211,7 @@ export class DatasetDAO {
}
// Extract all the directories structure and get the subdirectories for the required directory
if (wmode === Config.LS_MODE.ALL || wmode === Config.LS_MODE.DIRS) {
if (wmode !== Config.LS_MODE.DATASETS) {
const query = journalClient.createQuery(
Config.SEISMIC_STORE_NS + '-' + dataset.tenant + '-' + dataset.subproject, Config.DATASETS_KIND)
.select(['path']).groupBy('path').filter('path', '>', dataset.path).filter('path', '<', dataset.path + '\ufffd');
......@@ -202,7 +219,7 @@ export class DatasetDAO {
const [entitieshy] = await journalClient.runQuery(query);
results.directories = entitieshy.map((entity) => (entity.path as string).substr(dataset.path.length));
results.directories = results.directories.map(
(entity) => entity.substr(0, entity.indexOf('/'))).filter(
(entity) => entity.substr(0, entity.indexOf('/') + 1)).filter(
(elem, index, self) => index === self.indexOf(elem) );
}
......
src/services/utility/handler.ts
View file @ 62a9c102
// ============================================================================
// Copyright 2017-2020, Schlumberger
// Copyright 2017-2021, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -175,14 +175,14 @@ export class UtilityHandler {
if (pagination) {
// Retrieve paginated content list
return await DatasetDAO.paginatedListContent(journalClient, dataset, pagination);
return await DatasetDAO.paginatedListContent(journalClient, dataset, wmode, pagination);
}
// Retrieve complete content list
const results = await DatasetDAO.listContent(journalClient, dataset, wmode);
return (
(wmode === Config.LS_MODE.ALL || wmode === Config.LS_MODE.DIRS) ?
results.directories.map((el) => el.endsWith('/') ? el : el + '/') : []).concat(
results.directories : []).concat(
(wmode === Config.LS_MODE.ALL || wmode === Config.LS_MODE.DATASETS) ?
results.datasets : []);
}
......
src/services/utility/parser.ts
View file @ 62a9c102
// ============================================================================
// Copyright 2017-2019, Schlumberger
// Copyright 2017-2021, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -58,7 +58,7 @@ export class UtilityParser {
public static ls(req: expRequest): { sdPath: SDPathModel, wmode: string, pagination: PaginationModel } {
Params.checkString(req.query.sdpath, 'sdapth');
Params.checkString(req.query.sdpath, 'sdpath');
Params.checkString(req.query.wmode, 'wmode', false);
Params.checkString(req.query.limit, 'limit', false);
Params.checkString(req.query.cursor, 'cursor', false);
......@@ -98,7 +98,7 @@ export class UtilityParser {
public static gcsToken(req: expRequest): { sdPath: SDPathModel, readOnly: boolean } {
Params.checkString(req.query.sdpath, 'sdapth');
Params.checkString(req.query.sdpath, 'sdpath');
// extract the subproject path and ensure that is at least a subproject path
const sdPath = SDPath.getFromString(req.query.sdpath);
......
tests/utest/dao/dataset.ts
View file @ 62a9c102
// ============================================================================
// Copyright 2017-2019, Schlumberger
// Copyright 2017-2021 Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -476,7 +476,7 @@ export class TestDataset {
this.journal.runQuery.resolves([[{}], { endCursor: 'NO_MORE_RESULTS' }]);
await DatasetDAO.paginatedListContent(this.journal, this.dataset, pagination);
await DatasetDAO.paginatedListContent(this.journal, this.dataset, Config.LS_MODE.ALL, pagination);
Tx.checkTrue(this.journal.runQuery.calledWith(query), done);
});
......@@ -504,7 +504,7 @@ export class TestDataset {
this.journal.runQuery.resolves([[entityOne], { endCursor: 'MORE_RESULTS' }]);
await DatasetDAO.paginatedListContent(this.journal, this.dataset, pagination);
await DatasetDAO.paginatedListContent(this.journal, this.dataset, Config.LS_MODE.ALL, pagination);
Tx.checkTrue(this.journal.runQuery.calledWith(query), done);
});
......@@ -588,7 +588,7 @@ export class TestDataset {
Tx.checkTrue(
JSON.stringify(result.datasets) === JSON.stringify(
['dataset01']) && JSON.stringify(result.directories) === JSON.stringify(['a', 'd']), done);
['dataset01']) && JSON.stringify(result.directories) === JSON.stringify(['a/', 'd/']), done);
});
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment