Commit 8683fcb2 authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

Merge remote-tracking branch 'origin/master' into dev

parents 08c903b1 1c132537
......@@ -14,7 +14,6 @@ Apache-2.0
========================================================================
The following software have components provided under the terms of this license:
- @azure/core-tracing (from https://www.npmjs.com/package/@azure/core-tracing)
- @cloudant/cloudant (from https://www.npmjs.com/package/@cloudant/cloudant)
- @google-cloud/common (from https://www.npmjs.com/package/@google-cloud/common)
- @google-cloud/common (from https://www.npmjs.com/package/@google-cloud/common)
......@@ -38,16 +37,13 @@ The following software have components provided under the terms of this license:
- @opencensus/exporter-stackdriver (from https://www.npmjs.com/package/@opencensus/exporter-stackdriver)
- @opencensus/propagation-stackdriver (from https://www.npmjs.com/package/@opencensus/propagation-stackdriver)
- @opencensus/resource-util (from https://www.npmjs.com/package/@opencensus/resource-util)
- @opencensus/web-types (from https://www.npmjs.com/package/@opencensus/web-types)
- @opentelemetry/api (from https://www.npmjs.com/package/@opentelemetry/api)
- @opentelemetry/api (from https://www.npmjs.com/package/@opentelemetry/api)
- @opentelemetry/context-base (from https://www.npmjs.com/package/@opentelemetry/context-base)
- @opentelemetry/context-base (from https://www.npmjs.com/package/@opentelemetry/context-base)
- @opentelemetry/core (from https://www.npmjs.com/package/@opentelemetry/core)
- @opentelemetry/resources (from https://www.npmjs.com/package/@opentelemetry/resources)
- @opentelemetry/semantic-conventions (from https://www.npmjs.com/package/@opentelemetry/semantic-conventions)
- @opentelemetry/tracing (from https://www.npmjs.com/package/@opentelemetry/tracing)
- @opentelemetry/types (from https://www.npmjs.com/package/@opentelemetry/types)
- aws-sdk (from https://www.npmjs.com/package/aws-sdk)
- aws-sign2 (from https://www.npmjs.com/package/aws-sign2)
- boto3 (from https://github.com/boto/boto3)
......@@ -146,6 +142,7 @@ The following software have components provided under the terms of this license:
- tough-cookie (from https://www.npmjs.com/package/tough-cookie)
- tough-cookie (from https://www.npmjs.com/package/tough-cookie)
- tslib (from https://www.npmjs.com/package/tslib)
- tslib (from https://www.npmjs.com/package/tslib)
- uri-js (from https://www.npmjs.com/package/uri-js)
========================================================================
......@@ -359,15 +356,19 @@ The following software have components provided under the terms of this license:
- @azure/abort-controller (from https://www.npmjs.com/package/@azure/abort-controller)
- @azure/core-asynciterator-polyfill (from https://www.npmjs.com/package/@azure/core-asynciterator-polyfill)
- @azure/core-auth (from https://www.npmjs.com/package/@azure/core-auth)
- @azure/core-client (from https://www.npmjs.com/package/@azure/core-client)
- @azure/core-http (from https://www.npmjs.com/package/@azure/core-http)
- @azure/core-lro (from https://www.npmjs.com/package/@azure/core-lro)
- @azure/core-paging (from https://www.npmjs.com/package/@azure/core-paging)
- @azure/core-rest-pipeline (from https://www.npmjs.com/package/@azure/core-rest-pipeline)
- @azure/core-tracing (from https://www.npmjs.com/package/@azure/core-tracing)
- @azure/core-tracing (from https://www.npmjs.com/package/@azure/core-tracing)
- @azure/cosmos (from https://www.npmjs.com/package/@azure/cosmos)
- @azure/identity (from https://www.npmjs.com/package/@azure/identity)
- @azure/keyvault-secrets (from https://www.npmjs.com/package/@azure/keyvault-secrets)
- @azure/logger (from https://www.npmjs.com/package/@azure/logger)
- @azure/msal-common (from https://www.npmjs.com/package/@azure/msal-common)
- @azure/msal-node (from https://www.npmjs.com/package/@azure/msal-node)
- @azure/storage-blob (from https://www.npmjs.com/package/@azure/storage-blob)
- @dabh/diagnostics (from https://www.npmjs.com/package/@dabh/diagnostics)
- @tootallnate/once (from https://www.npmjs.com/package/@tootallnate/once)
......@@ -390,6 +391,7 @@ The following software have components provided under the terms of this license:
- @types/range-parser (from https://www.npmjs.com/package/@types/range-parser)
- @types/request (from https://www.npmjs.com/package/@types/request)
- @types/serve-static (from https://www.npmjs.com/package/@types/serve-static)
- @types/stoppable (from https://www.npmjs.com/package/@types/stoppable)
- @types/tough-cookie (from https://www.npmjs.com/package/@types/tough-cookie)
- @types/tunnel (from https://www.npmjs.com/package/@types/tunnel)
- PyJWT (from http://github.com/jpadilla/pyjwt)
......@@ -419,16 +421,19 @@ The following software have components provided under the terms of this license:
- aws4 (from https://www.npmjs.com/package/aws4)
- axios (from https://www.npmjs.com/package/axios)
- axios (from https://www.npmjs.com/package/axios)
- axios (from https://www.npmjs.com/package/axios)
- balanced-match (from https://www.npmjs.com/package/balanced-match)
- base64-js (from https://www.npmjs.com/package/base64-js)
- bcrypt-pbkdf (from https://www.npmjs.com/package/bcrypt-pbkdf)
- bignumber.js (from https://www.npmjs.com/package/bignumber.js)
- bl (from https://www.npmjs.com/package/bl)
- bluebird (from https://www.npmjs.com/package/bluebird)
- body-parser (from https://www.npmjs.com/package/body-parser)
- botocore (from https://github.com/boto/botocore)
- brace-expansion (from https://www.npmjs.com/package/brace-expansion)
- browser-request (from http://github.com/iriscouch/browser-request)
- buffer (from https://www.npmjs.com/package/buffer)
- buffer (from https://www.npmjs.com/package/buffer)
- buffer-from (from https://www.npmjs.com/package/buffer-from)
- builtin-modules (from https://www.npmjs.com/package/builtin-modules)
- bull (from https://www.npmjs.com/package/bull)
......@@ -468,7 +473,7 @@ The following software have components provided under the terms of this license:
- cross-spawn (from https://www.npmjs.com/package/cross-spawn)
- crypto-random-string (from https://www.npmjs.com/package/crypto-random-string)
- d64 (from https://www.npmjs.com/package/d64)
- dashdash (from https://github.com/trentm/node-dashdash)
- dashdash (from https://www.npmjs.com/package/dashdash)
- date-and-time (from https://www.npmjs.com/package/date-and-time)
- date-format (from https://www.npmjs.com/package/date-format)
- date-format (from https://www.npmjs.com/package/date-format)
......@@ -478,8 +483,10 @@ The following software have components provided under the terms of this license:
- debug (from https://www.npmjs.com/package/debug)
- debug (from https://www.npmjs.com/package/debug)
- debug (from https://www.npmjs.com/package/debug)
- debug (from https://www.npmjs.com/package/debug)
- debuglog (from https://www.npmjs.com/package/debuglog)
- decamelize (from https://www.npmjs.com/package/decamelize)
- decompress-response (from https://www.npmjs.com/package/decompress-response)
- deep-extend (from https://www.npmjs.com/package/deep-extend)
- define-properties (from https://www.npmjs.com/package/define-properties)
- delayed-stream (from https://www.npmjs.com/package/delayed-stream)
......@@ -509,6 +516,7 @@ The following software have components provided under the terms of this license:
- events (from https://www.npmjs.com/package/events)
- events (from https://www.npmjs.com/package/events)
- execa (from https://www.npmjs.com/package/execa)
- expand-template (from https://www.npmjs.com/package/expand-template)
- express (from https://www.npmjs.com/package/express)
- extend (from https://www.npmjs.com/package/extend)
- extsprintf (from https://www.npmjs.com/package/extsprintf)
......@@ -521,12 +529,15 @@ The following software have components provided under the terms of this license:
- flatted (from https://www.npmjs.com/package/flatted)
- fn.name (from https://www.npmjs.com/package/fn.name)
- follow-redirects (from https://www.npmjs.com/package/follow-redirects)
- follow-redirects (from https://www.npmjs.com/package/follow-redirects)
- for-each (from https://www.npmjs.com/package/for-each)
- form-data (from https://www.npmjs.com/package/form-data)
- form-data (from https://www.npmjs.com/package/form-data)
- form-data (from https://www.npmjs.com/package/form-data)
- form-data (from https://www.npmjs.com/package/form-data)
- forwarded (from https://www.npmjs.com/package/forwarded)
- fresh (from https://www.npmjs.com/package/fresh)
- fs-constants (from https://www.npmjs.com/package/fs-constants)
- fs-extra (from https://www.npmjs.com/package/fs-extra)
- fs-minipass (from https://www.npmjs.com/package/fs-minipass)
- fs.realpath (from https://github.com/isaacs/fs.realpath#readme)
......@@ -543,6 +554,7 @@ The following software have components provided under the terms of this license:
- get-stream (from https://www.npmjs.com/package/get-stream)
- get-stream (from https://www.npmjs.com/package/get-stream)
- getpass (from https://www.npmjs.com/package/getpass)
- github-from-package (from https://www.npmjs.com/package/github-from-package)
- glob (from https://www.npmjs.com/package/glob)
- glob (from https://www.npmjs.com/package/glob)
- google-p12-pem (from https://www.npmjs.com/package/google-p12-pem)
......@@ -578,6 +590,7 @@ The following software have components provided under the terms of this license:
- is-callable (from https://www.npmjs.com/package/is-callable)
- is-callable (from https://www.npmjs.com/package/is-callable)
- is-date-object (from https://www.npmjs.com/package/is-date-object)
- is-docker (from https://www.npmjs.com/package/is-docker)
- is-fullwidth-code-point (from https://www.npmjs.com/package/is-fullwidth-code-point)
- is-fullwidth-code-point (from https://www.npmjs.com/package/is-fullwidth-code-point)
- is-nan (from https://www.npmjs.com/package/is-nan)
......@@ -590,6 +603,7 @@ The following software have components provided under the terms of this license:
- is-stream-ended (from https://www.npmjs.com/package/is-stream-ended)
- is-symbol (from https://www.npmjs.com/package/is-symbol)
- is-typedarray (from https://github.com/hughsk/is-typedarray)
- is-wsl (from https://www.npmjs.com/package/is-wsl)
- isarray (from https://www.npmjs.com/package/isarray)
- isarray (from https://www.npmjs.com/package/isarray)
- isexe (from https://www.npmjs.com/package/isexe)
......@@ -610,6 +624,7 @@ The following software have components provided under the terms of this license:
- jwks-rsa (from https://www.npmjs.com/package/jwks-rsa)
- jws (from https://www.npmjs.com/package/jws)
- jws (from https://www.npmjs.com/package/jws)
- keytar (from https://www.npmjs.com/package/keytar)
- kuler (from https://www.npmjs.com/package/kuler)
- limiter (from https://www.npmjs.com/package/limiter)
- locate-path (from https://www.npmjs.com/package/locate-path)
......@@ -646,12 +661,14 @@ The following software have components provided under the terms of this license:
- mime-db (from https://www.npmjs.com/package/mime-db)
- mime-types (from https://www.npmjs.com/package/mime-types)
- mimic-fn (from https://www.npmjs.com/package/mimic-fn)
- mimic-response (from https://www.npmjs.com/package/mimic-response)
- minimatch (from https://www.npmjs.com/package/minimatch)
- minimist (from https://www.npmjs.com/package/minimist)
- minipass (from https://www.npmjs.com/package/minipass)
- minizlib (from https://www.npmjs.com/package/minizlib)
- mkdirp (from https://www.npmjs.com/package/mkdirp)
- mkdirp (from https://www.npmjs.com/package/mkdirp)
- mkdirp-classic (from https://www.npmjs.com/package/mkdirp-classic)
- module-details-from-path (from https://www.npmjs.com/package/module-details-from-path)
- moment (from https://www.npmjs.com/package/moment)
- moment-timezone (from https://www.npmjs.com/package/moment-timezone)
......@@ -660,11 +677,14 @@ The following software have components provided under the terms of this license:
- ms (from https://www.npmjs.com/package/ms)
- msal (from https://www.npmjs.com/package/msal)
- nan (from https://www.npmjs.com/package/nan)
- napi-build-utils (from https://www.npmjs.com/package/napi-build-utils)
- needle (from https://www.npmjs.com/package/needle)
- needle (from https://www.npmjs.com/package/needle)
- negotiator (from https://www.npmjs.com/package/negotiator)
- nice-try (from https://www.npmjs.com/package/nice-try)
- node-abi (from https://www.npmjs.com/package/node-abi)
- node-abort-controller (from https://www.npmjs.com/package/node-abort-controller)
- node-addon-api (from https://www.npmjs.com/package/node-addon-api)
- node-fetch (from https://www.npmjs.com/package/node-fetch)
- node-forge (from https://www.npmjs.com/package/node-forge)
- node-forge (from https://www.npmjs.com/package/node-forge)
......@@ -675,7 +695,7 @@ The following software have components provided under the terms of this license:
- npm-run-path (from https://www.npmjs.com/package/npm-run-path)
- npmlog (from https://www.npmjs.com/package/npmlog)
- number-is-nan (from https://github.com/sindresorhus/number-is-nan#readme)
- object-assign (from https://github.com/sindresorhus/object-assign#readme)
- object-assign (from https://www.npmjs.com/package/object-assign)
- object-hash (from https://www.npmjs.com/package/object-hash)
- object-inspect (from https://www.npmjs.com/package/object-inspect)
- object-inspect (from https://www.npmjs.com/package/object-inspect)
......@@ -687,6 +707,7 @@ The following software have components provided under the terms of this license:
- once (from https://github.com/isaacs/once#readme)
- one-time (from https://www.npmjs.com/package/one-time)
- onetime (from https://www.npmjs.com/package/onetime)
- open (from https://www.npmjs.com/package/open)
- os-homedir (from https://github.com/sindresorhus/os-homedir)
- os-name (from https://www.npmjs.com/package/os-name)
- os-tmpdir (from https://github.com/sindresorhus/os-tmpdir)
......@@ -706,6 +727,7 @@ The following software have components provided under the terms of this license:
- path-parse (from https://www.npmjs.com/package/path-parse)
- path-to-regexp (from https://www.npmjs.com/package/path-to-regexp)
- performance-now (from https://www.npmjs.com/package/performance-now)
- prebuild-install (from https://www.npmjs.com/package/prebuild-install)
- priorityqueuejs (from https://www.npmjs.com/package/priorityqueuejs)
- process (from https://www.npmjs.com/package/process)
- process-nextick-args (from https://www.npmjs.com/package/process-nextick-args)
......@@ -760,6 +782,8 @@ The following software have components provided under the terms of this license:
- shebang-command (from https://www.npmjs.com/package/shebang-command)
- shebang-regex (from https://www.npmjs.com/package/shebang-regex)
- signal-exit (from https://www.npmjs.com/package/signal-exit)
- simple-concat (from https://www.npmjs.com/package/simple-concat)
- simple-get (from https://www.npmjs.com/package/simple-get)
- simple-swizzle (from https://www.npmjs.com/package/simple-swizzle)
- six (from http://pypi.python.org/pypi/six/)
- snakecase-keys (from https://www.npmjs.com/package/snakecase-keys)
......@@ -772,6 +796,7 @@ The following software have components provided under the terms of this license:
- standard-as-callback (from https://www.npmjs.com/package/standard-as-callback)
- statuses (from https://www.npmjs.com/package/statuses)
- stealthy-require (from https://www.npmjs.com/package/stealthy-require)
- stoppable (from https://www.npmjs.com/package/stoppable)
- stream-events (from https://www.npmjs.com/package/stream-events)
- stream-shift (from https://www.npmjs.com/package/stream-shift)
- streamroller (from https://www.npmjs.com/package/streamroller)
......@@ -793,6 +818,8 @@ The following software have components provided under the terms of this license:
- swagger-ui-dist (from https://www.npmjs.com/package/swagger-ui-dist)
- swagger-ui-express (from https://www.npmjs.com/package/swagger-ui-express)
- tar (from https://www.npmjs.com/package/tar)
- tar-fs (from https://www.npmjs.com/package/tar-fs)
- tar-stream (from https://www.npmjs.com/package/tar-stream)
- text-hex (from https://www.npmjs.com/package/text-hex)
- through2 (from https://www.npmjs.com/package/through2)
- through2 (from https://www.npmjs.com/package/through2)
......@@ -804,6 +831,7 @@ The following software have components provided under the terms of this license:
- toidentifier (from https://www.npmjs.com/package/toidentifier)
- triple-beam (from https://www.npmjs.com/package/triple-beam)
- tslib (from https://www.npmjs.com/package/tslib)
- tslib (from https://www.npmjs.com/package/tslib)
- tunnel (from https://www.npmjs.com/package/tunnel)
- tweetnacl (from https://dchest.github.io/tweetnacl-js)
- type-fest (from https://www.npmjs.com/package/type-fest)
......@@ -930,4 +958,6 @@ The following software have components provided under the terms of this license:
- jmespath (from https://www.npmjs.com/package/jmespath)
- json-schema (from http://github.com/kriszyp/json-schema)
- querystring (from https://www.npmjs.com/package/querystring)
- sax (from https://www.npmjs.com/package/sax)
\ No newline at end of file
- sax (from https://www.npmjs.com/package/sax)
......@@ -2,9 +2,12 @@ global:
replicaCount: #{REPLICA_COUNT}#
namespace: osdu
podidentity: osdu-identity
nodepool: services
isAutoscalingEnabled: false
minReplicaCount: #{SDMS_MIN_REPLICAS}#
maxReplicaCount: #{SDMS_MAX_REPLICAS}#
configEnv:
cloudProvider: #{PROVIDER_NAME}#
keyvaultUrl: #{KEYVAULT_NAME}#
......@@ -17,4 +20,4 @@ configEnv:
image:
repository: #{CONTAINER_REGISTRY_NAME}#
branch: master
tag: #{IMAGE_TAG}#
\ No newline at end of file
tag: #{IMAGE_TAG}#
......@@ -15,7 +15,11 @@ spec:
labels:
app: {{ .Release.Name }}
aadpodidbinding: {{ .Values.global.podidentity }}
spec:
spec:
{{- if .Values.global.isAutoscalingEnabled }}
nodeSelector:
nodepool: {{ .Values.global.nodepool }}
{{- end }}
containers:
- name: {{ .Release.Name }}
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
......
......@@ -662,7 +662,14 @@ paths:
items:
type: string
collectionFormat: multi
- description: 'Limit the number of datasets in the response'
in: query
name: limit
type: string
- description: 'Cursor for pagination on the datasets list'
in: query
name: cursor
type: string
responses:
200:
description: "The list of all datasets in the subproject if no gtags are in the request parameters. If gtags exist in the request parameters, then list all datasets that have the same list of gtags."
......@@ -670,6 +677,10 @@ paths:
type: array
items:
$ref: "#/definitions/Dataset"
201:
description: "Paginated dataset list with nextPageCursor. For documentation purposes, if limit or cursor is given, status code here is 200."
schema:
$ref: "#/definitions/PaginatedDatasets"
400:
description: "Bad request."
401:
......@@ -2196,6 +2207,16 @@ definitions:
type: string
description: Next cursor for pagination.
example: { datasets: ["folderA/", "folderB/", "dataset01"], nextPageCursor: "abc1234" }
# PaginatedDatasets
PaginatedDatasets:
properties:
datasets:
type: array
items:
$ref: '#/definitions/Dataset'
nextPageCursor:
type: string
# ===========================================================================
# Endpoints Security Section
......
......@@ -672,6 +672,14 @@ paths:
items:
type: string
collectionFormat: multi
- description: 'Limit the number of datasets in the response'
in: query
name: limit
type: string
- description: 'Cursor for pagination on the datasets list'
in: query
name: cursor
type: string
responses:
200:
......@@ -680,6 +688,11 @@ paths:
type: array
items:
$ref: "#/definitions/Dataset"
201:
description: "Paginated dataset list with nextPageCursor. For documentation purposes, if limit or cursor is given, status code here is 200."
schema:
$ref: "#/definitions/PaginatedDatasets"
400:
description: "Bad request"
401:
......@@ -2205,6 +2218,16 @@ definitions:
type: string
description: Next cursor for pagination
example: { datasets: ["folderA/", "folderB/", "dataset01"], nextPageCursor: "abc1234" }
# PaginatedDatasets
PaginatedDatasets:
properties:
datasets:
type: array
items:
$ref: '#/definitions/Dataset'
nextPageCursor:
type: string
# ===========================================================================
# Endpoints Security Section
......
This diff is collapsed.
......@@ -27,7 +27,7 @@ export class ConfigGoogle extends Config {
// endpoints
public static GOOGLE_EP_IAM = 'https://iam.googleapis.com/v1';
public static GOOGLE_EP_OAUTH2 = 'https://www.googleapis.com/oauth2/v4';
public static GOOGLE_EP_METADATA = 'http://metadata/computeMetadata/v1';
public static GOOGLE_EP_METADATA = 'http://169.254.169.254/computeMetadata/v1';
public static GOOGLE_EP_ROBOT = 'https://www.googleapis.com/robot/v1';
public static GOOGLE_EP_RESOURCES = 'https://cloudresourcemanager.googleapis.com/v1';
......
......@@ -15,10 +15,10 @@
// ============================================================================
import { DatasetModel, PaginationModel } from '.';
import { IJournal, IJournalTransaction } from '../../cloud';
import { Config } from '../../cloud';
import { Config, IJournal, IJournalTransaction } from '../../cloud';
import { Utils } from '../../shared';
import { Locker } from './locker';
import { PaginatedDatasetList } from './model';
export class DatasetDAO {
......@@ -69,7 +69,9 @@ export class DatasetDAO {
public static async list(
journalClient: IJournal | IJournalTransaction,
dataset: DatasetModel): Promise<DatasetModel[]> {
dataset: DatasetModel, pagination: PaginationModel):
Promise<any> {
let query: any;
if (dataset.gtags === undefined || dataset.gtags.length === 0) {
query = journalClient.createQuery(
......@@ -83,16 +85,27 @@ export class DatasetDAO {
}
}
let [entities] = await journalClient.runQuery(query);
if (pagination && pagination.cursor) { query = query.start(pagination.cursor); }
if (pagination && pagination.limit) { query = query.limit(pagination.limit); }
entities = (entities) as DatasetModel[];
const [entities, info] = await journalClient.runQuery(query);
// Fix model for old entity
for (let entity of entities) {
entity = await this.fixOldModel(entity, dataset.tenant, dataset.subproject);
}
return entities;
const output: PaginatedDatasetList = {
datasets: entities,
nextPageCursor: null
};
if (pagination) {
output.nextPageCursor = info.endCursor;
}
return output;
}
......
......@@ -318,7 +318,10 @@ export class DatasetHandler {
private static async list(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
// Retrieve the dataset path information
const dataset = DatasetParser.list(req);
const userInput = DatasetParser.list(req);
const dataset = userInput.dataset;
const pagination = userInput.pagination;
// init journalClient client
const journalClient = JournalFactoryTenantClient.get(tenant);
......@@ -331,18 +334,21 @@ export class DatasetHandler {
req.headers['impersonation-token-context'] as string);
}
// Retrieve the list of datasets metadata
const datasets = await DatasetDAO.list(journalClient, dataset);
const output = await DatasetDAO.list(journalClient, dataset, pagination) as any;
// attach the gcpid for fast check
for (const item of datasets) {
for (const item of output.datasets) {
item.ctag = item.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);
}
// Retrieve the list of datasets metadata
return datasets;
if (output.nextPageCursor) {
return output;
}
return output.datasets;
}
// delete a dataset
......@@ -834,7 +840,7 @@ export class DatasetHandler {
// check if the dataset does not exist
const lockKey = datasetIN.tenant + '/' + datasetIN.subproject + datasetIN.path + datasetIN.name;
if (!dataset) {
if(await Locker.getLock(lockKey)) {
if (await Locker.getLock(lockKey)) {
// if a previous call fails, the dataset is not created but the lock is acquired and not released
await Locker.unlock(lockKey);
return;
......
......@@ -14,6 +14,7 @@
// limitations under the License.
// ============================================================================
export interface IDatasetModel {
name: string;
tenant: string;
......@@ -46,4 +47,9 @@ export interface IPaginationModel {
export interface IDatasetAcl {
admins: string[],
viewers: string[];
}
export interface PaginatedDatasetList {
datasets: IDatasetModel[];
nextPageCursor: string;
}
\ No newline at end of file
......@@ -109,10 +109,28 @@ export class DatasetParser {
req.query.seismicmeta === 'true'];
}
public static list(req: expRequest): DatasetModel {
public static list(req: expRequest): any {
const dataset = this.createDatasetModelFromRequest(req);
dataset.gtags = req.query.gtag;
return dataset;
const limit = parseInt(req.query.limit, 10);
if (limit < 0) {
throw (Error.make(Error.Status.BAD_REQUEST,
'The \'limit\' query parameter can not be less than zero.'));
}
const cursor = req.query.cursor as string;
if (cursor === '') {
throw (Error.make(Error.Status.BAD_REQUEST,
'The \'cursor\' query parameter can not be empty if supplied'));
}
let pagination = null;
if (limit || cursor) {
pagination = { limit, cursor };
}
// Retrieve the list of datasets metadata
return { dataset, pagination };
}
public static delete(req: expRequest): DatasetModel {
......
......@@ -91,7 +91,11 @@ export class ImpTokenDAO {
try {
await request(options);
} catch (error) {
throw (Error.make(Error.Status.BAD_REQUEST, 'The impersonation token cannot be refreshed.'));
// For any code different than 4xx the imptoken can be refreshed
// This is a temporary fix to handle unavailability of client infrastructure
if (error.statusCode >= 400 && error.statusCode <= 499) {
throw (Error.make(Error.Status.BAD_REQUEST, 'The impersonation token cannot be refreshed.'));
}
}
}
......
......@@ -14,13 +14,12 @@
// limitations under the License.
// ============================================================================
import sinon from 'sinon';
import { Datastore } from '@google-cloud/datastore';
import { Entity } from '@google-cloud/datastore/build/src/entity';
import { RunQueryResponse } from '@google-cloud/datastore/build/src/query';
import { google } from '../../../src/cloud/providers';
import sinon from 'sinon';
import { Config } from '../../../src/cloud';
import { google } from '../../../src/cloud/providers';
import { RecordLatency } from '../../../src/metrics';
import { DatasetModel } from '../../../src/services/dataset';
import { DatasetDAO } from '../../../src/services/dataset/dao';
......@@ -28,6 +27,7 @@ import { Locker } from '../../../src/services/dataset/locker';
import { IPaginationModel } from '../../../src/services/dataset/model';
import { Tx } from '../utils';
export class TestDataset {
private static dataset: DatasetModel;
......@@ -82,13 +82,13 @@ export class TestDataset {
Tx.test(async (done: any) => {
this.journal.save.resolves({} as never);
await DatasetDAO.register(this.journal, { key: {'key': 'dataset_key'}, data: TestDataset.dataset });
await DatasetDAO.register(this.journal, { key: { 'key': 'dataset_key' }, data: TestDataset.dataset });
done();
});
Tx.test(async (done: any) => {
this.journal.save.resolves();
await DatasetDAO.register(this.journal, { key: {'key': 'dataset_key'}, data: TestDataset.dataset });
await DatasetDAO.register(this.journal, { key: { 'key': 'dataset_key' }, data: TestDataset.dataset });
done();
});
}
......@@ -186,7 +186,7 @@ export class TestDataset {
this.journal.runQuery.resolves([expectedResult, undefined]);
this.sandbox.stub(DatasetDAO, 'fixOldModel').resolves(expectedResult[0]);
const result = await DatasetDAO.list(this.journal, this.dataset);
const result = await DatasetDAO.list(this.journal, this.dataset, null);
Tx.checkTrue(
this.journal.runQuery.calledWith(query) && result[0] === expectedResult[0],
......@@ -231,7 +231,7 @@ export class TestDataset {
this.journal.runQuery.resolves([expectedResult, undefined]);
this.sandbox.stub(DatasetDAO, 'fixOldModel').resolves(expectedResult[0]);
const result = await DatasetDAO.list(this.journal, this.dataset);
const result = await DatasetDAO.list(this.journal, this.dataset, null);
Tx.checkTrue(
this.journal.runQuery.calledWith(query) && result[0] === expectedResult[0],
......@@ -274,7 +274,7 @@ export class TestDataset {
this.journal.getQueryFilterSymbolContains.returns('=');
this.sandbox.stub(DatasetDAO, 'fixOldModel').resolves(expectedResult[0]);
const result = await DatasetDAO.list(this.journal, this.dataset);
const result = await DatasetDAO.list(this.journal, this.dataset, null);
Tx.checkTrue(
this.journal.runQuery.calledWith(query) && result[0] === expectedResult[0],