Merge remote-tracking branch 'origin/master' into dev

NOTICE

@@ -14,7 +14,6 @@ Apache-2.0
 ========================================================================
 The following software have components provided under the terms of this license:
 
-- @azure/core-tracing (from https://www.npmjs.com/package/@azure/core-tracing)
 - @cloudant/cloudant (from https://www.npmjs.com/package/@cloudant/cloudant)
 - @google-cloud/common (from https://www.npmjs.com/package/@google-cloud/common)
 - @google-cloud/common (from https://www.npmjs.com/package/@google-cloud/common)
@@ -38,16 +37,13 @@ The following software have components provided under the terms of this license:
 - @opencensus/exporter-stackdriver (from https://www.npmjs.com/package/@opencensus/exporter-stackdriver)
 - @opencensus/propagation-stackdriver (from https://www.npmjs.com/package/@opencensus/propagation-stackdriver)
 - @opencensus/resource-util (from https://www.npmjs.com/package/@opencensus/resource-util)
-- @opencensus/web-types (from https://www.npmjs.com/package/@opencensus/web-types)
 - @opentelemetry/api (from https://www.npmjs.com/package/@opentelemetry/api)
 - @opentelemetry/api (from https://www.npmjs.com/package/@opentelemetry/api)
 - @opentelemetry/context-base (from https://www.npmjs.com/package/@opentelemetry/context-base)
-- @opentelemetry/context-base (from https://www.npmjs.com/package/@opentelemetry/context-base)
 - @opentelemetry/core (from https://www.npmjs.com/package/@opentelemetry/core)
 - @opentelemetry/resources (from https://www.npmjs.com/package/@opentelemetry/resources)
 - @opentelemetry/semantic-conventions (from https://www.npmjs.com/package/@opentelemetry/semantic-conventions)
 - @opentelemetry/tracing (from https://www.npmjs.com/package/@opentelemetry/tracing)
-- @opentelemetry/types (from https://www.npmjs.com/package/@opentelemetry/types)
 - aws-sdk (from https://www.npmjs.com/package/aws-sdk)
 - aws-sign2 (from https://www.npmjs.com/package/aws-sign2)
 - boto3 (from https://github.com/boto/boto3)
@@ -146,6 +142,7 @@ The following software have components provided under the terms of this license:
 - tough-cookie (from https://www.npmjs.com/package/tough-cookie)
 - tough-cookie (from https://www.npmjs.com/package/tough-cookie)
 - tslib (from https://www.npmjs.com/package/tslib)
+- tslib (from https://www.npmjs.com/package/tslib)
 - uri-js (from https://www.npmjs.com/package/uri-js)
 
 ========================================================================
@@ -359,15 +356,19 @@ The following software have components provided under the terms of this license:
 - @azure/abort-controller (from https://www.npmjs.com/package/@azure/abort-controller)
 - @azure/core-asynciterator-polyfill (from https://www.npmjs.com/package/@azure/core-asynciterator-polyfill)
 - @azure/core-auth (from https://www.npmjs.com/package/@azure/core-auth)
+- @azure/core-client (from https://www.npmjs.com/package/@azure/core-client)
 - @azure/core-http (from https://www.npmjs.com/package/@azure/core-http)
 - @azure/core-lro (from https://www.npmjs.com/package/@azure/core-lro)
 - @azure/core-paging (from https://www.npmjs.com/package/@azure/core-paging)
+- @azure/core-rest-pipeline (from https://www.npmjs.com/package/@azure/core-rest-pipeline)
 - @azure/core-tracing (from https://www.npmjs.com/package/@azure/core-tracing)
 - @azure/core-tracing (from https://www.npmjs.com/package/@azure/core-tracing)
 - @azure/cosmos (from https://www.npmjs.com/package/@azure/cosmos)
 - @azure/identity (from https://www.npmjs.com/package/@azure/identity)
 - @azure/keyvault-secrets (from https://www.npmjs.com/package/@azure/keyvault-secrets)
 - @azure/logger (from https://www.npmjs.com/package/@azure/logger)
+- @azure/msal-common (from https://www.npmjs.com/package/@azure/msal-common)
+- @azure/msal-node (from https://www.npmjs.com/package/@azure/msal-node)
 - @azure/storage-blob (from https://www.npmjs.com/package/@azure/storage-blob)
 - @dabh/diagnostics (from https://www.npmjs.com/package/@dabh/diagnostics)
 - @tootallnate/once (from https://www.npmjs.com/package/@tootallnate/once)
@@ -390,6 +391,7 @@ The following software have components provided under the terms of this license:
 - @types/range-parser (from https://www.npmjs.com/package/@types/range-parser)
 - @types/request (from https://www.npmjs.com/package/@types/request)
 - @types/serve-static (from https://www.npmjs.com/package/@types/serve-static)
+- @types/stoppable (from https://www.npmjs.com/package/@types/stoppable)
 - @types/tough-cookie (from https://www.npmjs.com/package/@types/tough-cookie)
 - @types/tunnel (from https://www.npmjs.com/package/@types/tunnel)
 - PyJWT (from http://github.com/jpadilla/pyjwt)
@@ -419,16 +421,19 @@ The following software have components provided under the terms of this license:
 - aws4 (from https://www.npmjs.com/package/aws4)
 - axios (from https://www.npmjs.com/package/axios)
 - axios (from https://www.npmjs.com/package/axios)
+- axios (from https://www.npmjs.com/package/axios)
 - balanced-match (from https://www.npmjs.com/package/balanced-match)
 - base64-js (from https://www.npmjs.com/package/base64-js)
 - bcrypt-pbkdf (from https://www.npmjs.com/package/bcrypt-pbkdf)
 - bignumber.js (from https://www.npmjs.com/package/bignumber.js)
+- bl (from https://www.npmjs.com/package/bl)
 - bluebird (from https://www.npmjs.com/package/bluebird)
 - body-parser (from https://www.npmjs.com/package/body-parser)
 - botocore (from https://github.com/boto/botocore)
 - brace-expansion (from https://www.npmjs.com/package/brace-expansion)
 - browser-request (from http://github.com/iriscouch/browser-request)
 - buffer (from https://www.npmjs.com/package/buffer)
+- buffer (from https://www.npmjs.com/package/buffer)
 - buffer-from (from https://www.npmjs.com/package/buffer-from)
 - builtin-modules (from https://www.npmjs.com/package/builtin-modules)
 - bull (from https://www.npmjs.com/package/bull)
@@ -468,7 +473,7 @@ The following software have components provided under the terms of this license:
 - cross-spawn (from https://www.npmjs.com/package/cross-spawn)
 - crypto-random-string (from https://www.npmjs.com/package/crypto-random-string)
 - d64 (from https://www.npmjs.com/package/d64)
-- dashdash (from https://github.com/trentm/node-dashdash)
+- dashdash (from https://www.npmjs.com/package/dashdash)
 - date-and-time (from https://www.npmjs.com/package/date-and-time)
 - date-format (from https://www.npmjs.com/package/date-format)
 - date-format (from https://www.npmjs.com/package/date-format)
@@ -478,8 +483,10 @@ The following software have components provided under the terms of this license:
 - debug (from https://www.npmjs.com/package/debug)
 - debug (from https://www.npmjs.com/package/debug)
 - debug (from https://www.npmjs.com/package/debug)
+- debug (from https://www.npmjs.com/package/debug)
 - debuglog (from https://www.npmjs.com/package/debuglog)
 - decamelize (from https://www.npmjs.com/package/decamelize)
+- decompress-response (from https://www.npmjs.com/package/decompress-response)
 - deep-extend (from https://www.npmjs.com/package/deep-extend)
 - define-properties (from https://www.npmjs.com/package/define-properties)
 - delayed-stream (from https://www.npmjs.com/package/delayed-stream)
@@ -509,6 +516,7 @@ The following software have components provided under the terms of this license:
 - events (from https://www.npmjs.com/package/events)
 - events (from https://www.npmjs.com/package/events)
 - execa (from https://www.npmjs.com/package/execa)
+- expand-template (from https://www.npmjs.com/package/expand-template)
 - express (from https://www.npmjs.com/package/express)
 - extend (from https://www.npmjs.com/package/extend)
 - extsprintf (from https://www.npmjs.com/package/extsprintf)
@@ -521,12 +529,15 @@ The following software have components provided under the terms of this license:
 - flatted (from https://www.npmjs.com/package/flatted)
 - fn.name (from https://www.npmjs.com/package/fn.name)
 - follow-redirects (from https://www.npmjs.com/package/follow-redirects)
+- follow-redirects (from https://www.npmjs.com/package/follow-redirects)
 - for-each (from https://www.npmjs.com/package/for-each)
 - form-data (from https://www.npmjs.com/package/form-data)
 - form-data (from https://www.npmjs.com/package/form-data)
 - form-data (from https://www.npmjs.com/package/form-data)
+- form-data (from https://www.npmjs.com/package/form-data)
 - forwarded (from https://www.npmjs.com/package/forwarded)
 - fresh (from https://www.npmjs.com/package/fresh)
+- fs-constants (from https://www.npmjs.com/package/fs-constants)
 - fs-extra (from https://www.npmjs.com/package/fs-extra)
 - fs-minipass (from https://www.npmjs.com/package/fs-minipass)
 - fs.realpath (from https://github.com/isaacs/fs.realpath#readme)
@@ -543,6 +554,7 @@ The following software have components provided under the terms of this license:
 - get-stream (from https://www.npmjs.com/package/get-stream)
 - get-stream (from https://www.npmjs.com/package/get-stream)
 - getpass (from https://www.npmjs.com/package/getpass)
+- github-from-package (from https://www.npmjs.com/package/github-from-package)
 - glob (from https://www.npmjs.com/package/glob)
 - glob (from https://www.npmjs.com/package/glob)
 - google-p12-pem (from https://www.npmjs.com/package/google-p12-pem)
@@ -578,6 +590,7 @@ The following software have components provided under the terms of this license:
 - is-callable (from https://www.npmjs.com/package/is-callable)
 - is-callable (from https://www.npmjs.com/package/is-callable)
 - is-date-object (from https://www.npmjs.com/package/is-date-object)
+- is-docker (from https://www.npmjs.com/package/is-docker)
 - is-fullwidth-code-point (from https://www.npmjs.com/package/is-fullwidth-code-point)
 - is-fullwidth-code-point (from https://www.npmjs.com/package/is-fullwidth-code-point)
 - is-nan (from https://www.npmjs.com/package/is-nan)
@@ -590,6 +603,7 @@ The following software have components provided under the terms of this license:
 - is-stream-ended (from https://www.npmjs.com/package/is-stream-ended)
 - is-symbol (from https://www.npmjs.com/package/is-symbol)
 - is-typedarray (from https://github.com/hughsk/is-typedarray)
+- is-wsl (from https://www.npmjs.com/package/is-wsl)
 - isarray (from https://www.npmjs.com/package/isarray)
 - isarray (from https://www.npmjs.com/package/isarray)
 - isexe (from https://www.npmjs.com/package/isexe)
@@ -610,6 +624,7 @@ The following software have components provided under the terms of this license:
 - jwks-rsa (from https://www.npmjs.com/package/jwks-rsa)
 - jws (from https://www.npmjs.com/package/jws)
 - jws (from https://www.npmjs.com/package/jws)
+- keytar (from https://www.npmjs.com/package/keytar)
 - kuler (from https://www.npmjs.com/package/kuler)
 - limiter (from https://www.npmjs.com/package/limiter)
 - locate-path (from https://www.npmjs.com/package/locate-path)
@@ -646,12 +661,14 @@ The following software have components provided under the terms of this license:
 - mime-db (from https://www.npmjs.com/package/mime-db)
 - mime-types (from https://www.npmjs.com/package/mime-types)
 - mimic-fn (from https://www.npmjs.com/package/mimic-fn)
+- mimic-response (from https://www.npmjs.com/package/mimic-response)
 - minimatch (from https://www.npmjs.com/package/minimatch)
 - minimist (from https://www.npmjs.com/package/minimist)
 - minipass (from https://www.npmjs.com/package/minipass)
 - minizlib (from https://www.npmjs.com/package/minizlib)
 - mkdirp (from https://www.npmjs.com/package/mkdirp)
 - mkdirp (from https://www.npmjs.com/package/mkdirp)
+- mkdirp-classic (from https://www.npmjs.com/package/mkdirp-classic)
 - module-details-from-path (from https://www.npmjs.com/package/module-details-from-path)
 - moment (from https://www.npmjs.com/package/moment)
 - moment-timezone (from https://www.npmjs.com/package/moment-timezone)
@@ -660,11 +677,14 @@ The following software have components provided under the terms of this license:
 - ms (from https://www.npmjs.com/package/ms)
 - msal (from https://www.npmjs.com/package/msal)
 - nan (from https://www.npmjs.com/package/nan)
+- napi-build-utils (from https://www.npmjs.com/package/napi-build-utils)
 - needle (from https://www.npmjs.com/package/needle)
 - needle (from https://www.npmjs.com/package/needle)
 - negotiator (from https://www.npmjs.com/package/negotiator)
 - nice-try (from https://www.npmjs.com/package/nice-try)
+- node-abi (from https://www.npmjs.com/package/node-abi)
 - node-abort-controller (from https://www.npmjs.com/package/node-abort-controller)
+- node-addon-api (from https://www.npmjs.com/package/node-addon-api)
 - node-fetch (from https://www.npmjs.com/package/node-fetch)
 - node-forge (from https://www.npmjs.com/package/node-forge)
 - node-forge (from https://www.npmjs.com/package/node-forge)
@@ -675,7 +695,7 @@ The following software have components provided under the terms of this license:
 - npm-run-path (from https://www.npmjs.com/package/npm-run-path)
 - npmlog (from https://www.npmjs.com/package/npmlog)
 - number-is-nan (from https://github.com/sindresorhus/number-is-nan#readme)
-- object-assign (from https://github.com/sindresorhus/object-assign#readme)
+- object-assign (from https://www.npmjs.com/package/object-assign)
 - object-hash (from https://www.npmjs.com/package/object-hash)
 - object-inspect (from https://www.npmjs.com/package/object-inspect)
 - object-inspect (from https://www.npmjs.com/package/object-inspect)
@@ -687,6 +707,7 @@ The following software have components provided under the terms of this license:
 - once (from https://github.com/isaacs/once#readme)
 - one-time (from https://www.npmjs.com/package/one-time)
 - onetime (from https://www.npmjs.com/package/onetime)
+- open (from https://www.npmjs.com/package/open)
 - os-homedir (from https://github.com/sindresorhus/os-homedir)
 - os-name (from https://www.npmjs.com/package/os-name)
 - os-tmpdir (from https://github.com/sindresorhus/os-tmpdir)
@@ -706,6 +727,7 @@ The following software have components provided under the terms of this license:
 - path-parse (from https://www.npmjs.com/package/path-parse)
 - path-to-regexp (from https://www.npmjs.com/package/path-to-regexp)
 - performance-now (from https://www.npmjs.com/package/performance-now)
+- prebuild-install (from https://www.npmjs.com/package/prebuild-install)
 - priorityqueuejs (from https://www.npmjs.com/package/priorityqueuejs)
 - process (from https://www.npmjs.com/package/process)
 - process-nextick-args (from https://www.npmjs.com/package/process-nextick-args)
@@ -760,6 +782,8 @@ The following software have components provided under the terms of this license:
 - shebang-command (from https://www.npmjs.com/package/shebang-command)
 - shebang-regex (from https://www.npmjs.com/package/shebang-regex)
 - signal-exit (from https://www.npmjs.com/package/signal-exit)
+- simple-concat (from https://www.npmjs.com/package/simple-concat)
+- simple-get (from https://www.npmjs.com/package/simple-get)
 - simple-swizzle (from https://www.npmjs.com/package/simple-swizzle)
 - six (from http://pypi.python.org/pypi/six/)
 - snakecase-keys (from https://www.npmjs.com/package/snakecase-keys)
@@ -772,6 +796,7 @@ The following software have components provided under the terms of this license:
 - standard-as-callback (from https://www.npmjs.com/package/standard-as-callback)
 - statuses (from https://www.npmjs.com/package/statuses)
 - stealthy-require (from https://www.npmjs.com/package/stealthy-require)
+- stoppable (from https://www.npmjs.com/package/stoppable)
 - stream-events (from https://www.npmjs.com/package/stream-events)
 - stream-shift (from https://www.npmjs.com/package/stream-shift)
 - streamroller (from https://www.npmjs.com/package/streamroller)
@@ -793,6 +818,8 @@ The following software have components provided under the terms of this license:
 - swagger-ui-dist (from https://www.npmjs.com/package/swagger-ui-dist)
 - swagger-ui-express (from https://www.npmjs.com/package/swagger-ui-express)
 - tar (from https://www.npmjs.com/package/tar)
+- tar-fs (from https://www.npmjs.com/package/tar-fs)
+- tar-stream (from https://www.npmjs.com/package/tar-stream)
 - text-hex (from https://www.npmjs.com/package/text-hex)
 - through2 (from https://www.npmjs.com/package/through2)
 - through2 (from https://www.npmjs.com/package/through2)
@@ -804,6 +831,7 @@ The following software have components provided under the terms of this license:
 - toidentifier (from https://www.npmjs.com/package/toidentifier)
 - triple-beam (from https://www.npmjs.com/package/triple-beam)
 - tslib (from https://www.npmjs.com/package/tslib)
+- tslib (from https://www.npmjs.com/package/tslib)
 - tunnel (from https://www.npmjs.com/package/tunnel)
 - tweetnacl (from https://dchest.github.io/tweetnacl-js)
 - type-fest (from https://www.npmjs.com/package/type-fest)
@@ -930,4 +958,6 @@ The following software have components provided under the terms of this license:
 - jmespath (from https://www.npmjs.com/package/jmespath)
 - json-schema (from http://github.com/kriszyp/json-schema)
 - querystring (from https://www.npmjs.com/package/querystring)
-- sax (from https://www.npmjs.com/package/sax)
\ No newline at end of file
+- sax (from https://www.npmjs.com/package/sax)
+
+

devops/azure/chart/helm-config.yaml

@@ -2,9 +2,12 @@ global:
   replicaCount: #{REPLICA_COUNT}#
   namespace: osdu
   podidentity: osdu-identity
+  nodepool: services
+  isAutoscalingEnabled: false
   minReplicaCount: #{SDMS_MIN_REPLICAS}#
   maxReplicaCount: #{SDMS_MAX_REPLICAS}#
  
+ 
 configEnv:
   cloudProvider: #{PROVIDER_NAME}#
   keyvaultUrl: #{KEYVAULT_NAME}#
@@ -17,4 +20,4 @@ configEnv:
 image:
   repository: #{CONTAINER_REGISTRY_NAME}#
   branch: master
-  tag: #{IMAGE_TAG}#
\ No newline at end of file
+  tag: #{IMAGE_TAG}#

devops/azure/chart/templates/deployment.yaml

@@ -15,7 +15,11 @@ spec:
       labels:
         app: {{ .Release.Name }}
         aadpodidbinding: {{ .Values.global.podidentity }}
-    spec:
+    spec: 
+{{- if .Values.global.isAutoscalingEnabled }}
+      nodeSelector:
+        nodepool: {{ .Values.global.nodepool }} 
+{{- end }}   
       containers:
         - name: {{ .Release.Name }}
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}

docs/api/openapi.osdu.yaml

@@ -662,7 +662,14 @@ paths:
           items:
             type: string
           collectionFormat: multi
-
+        - description: 'Limit the number of datasets in the response'
+          in: query
+          name: limit
+          type: string 
+        - description: 'Cursor for pagination on the datasets list'
+          in: query
+          name: cursor
+          type: string 
       responses:
         200:
           description: "The list of all datasets in the subproject if no gtags are in the request parameters. If gtags exist in the request parameters, then list all datasets that have the same list of gtags."
@@ -670,6 +677,10 @@ paths:
             type: array
             items:
               $ref: "#/definitions/Dataset"
+        201:
+          description: "Paginated dataset list with nextPageCursor. For documentation purposes, if limit or cursor is given, status code here is 200."
+          schema:
+            $ref: "#/definitions/PaginatedDatasets"
         400:
           description: "Bad request."
         401:
@@ -2196,6 +2207,16 @@ definitions:
         type: string
         description: Next cursor for pagination.
     example: { datasets: ["folderA/", "folderB/", "dataset01"], nextPageCursor: "abc1234" }
+    
+ # PaginatedDatasets
+  PaginatedDatasets:
+    properties:
+        datasets:
+          type: array
+          items:
+            $ref: '#/definitions/Dataset'
+        nextPageCursor:
+            type: string 
 
 # ===========================================================================
 # Endpoints Security Section

docs/api/openapi.yaml

@@ -672,6 +672,14 @@ paths:
           items:
             type: string
           collectionFormat: multi
+        - description: 'Limit the number of datasets in the response'
+          in: query
+          name: limit
+          type: string 
+        - description: 'Cursor for pagination on the datasets list'
+          in: query
+          name: cursor
+          type: string 
 
       responses:
         200:
@@ -680,6 +688,11 @@ paths:
             type: array
             items:
               $ref: "#/definitions/Dataset"
+              
+        201:
+          description: "Paginated dataset list with nextPageCursor. For documentation purposes, if limit or cursor is given, status code here is 200."
+          schema:
+            $ref: "#/definitions/PaginatedDatasets"
         400:
           description: "Bad request"
         401:
@@ -2205,6 +2218,16 @@ definitions:
         type: string
         description: Next cursor for pagination
     example: { datasets: ["folderA/", "folderB/", "dataset01"], nextPageCursor: "abc1234" }
+    
+   # PaginatedDatasets
+  PaginatedDatasets:
+    properties:
+        datasets:
+          type: array
+          items:
+            $ref: '#/definitions/Dataset'
+        nextPageCursor:
+            type: string 
 
 # ===========================================================================
 # Endpoints Security Section

package.json

@@ -50,9 +50,9 @@
   },
   "dependencies": {
     "@azure/cosmos": "^3.7.2",
-    "@azure/identity": "^1.0.3",
-    "@azure/keyvault-secrets": "^4.0.4",
-    "@azure/storage-blob": "^12.1.2",
+    "@azure/identity": "^1.5.0",
+    "@azure/keyvault-secrets": "^4.3.0",
+    "@azure/storage-blob": "^12.7.0",
     "@cloudant/cloudant": "^4.2.4",
     "@google-cloud/datastore": "6.3.1",
     "@google-cloud/logging": "^9.0.0",

src/cloud/providers/google/config.ts

@@ -27,7 +27,7 @@ export class ConfigGoogle extends Config {
     // endpoints
     public static GOOGLE_EP_IAM = 'https://iam.googleapis.com/v1';
     public static GOOGLE_EP_OAUTH2 = 'https://www.googleapis.com/oauth2/v4';
-    public static GOOGLE_EP_METADATA = 'http://metadata/computeMetadata/v1';
+    public static GOOGLE_EP_METADATA = 'http://169.254.169.254/computeMetadata/v1';
     public static GOOGLE_EP_ROBOT = 'https://www.googleapis.com/robot/v1';
     public static GOOGLE_EP_RESOURCES = 'https://cloudresourcemanager.googleapis.com/v1';
 

src/services/dataset/dao.ts

@@ -15,10 +15,10 @@
 // ============================================================================
 
 import { DatasetModel, PaginationModel } from '.';
-import { IJournal, IJournalTransaction } from '../../cloud';
-import { Config } from '../../cloud';
+import { Config, IJournal, IJournalTransaction } from '../../cloud';
 import { Utils } from '../../shared';
 import { Locker } from './locker';
+import { PaginatedDatasetList } from './model';
 
 export class DatasetDAO {
 
@@ -69,7 +69,9 @@ export class DatasetDAO {
 
     public static async list(
         journalClient: IJournal | IJournalTransaction,
-        dataset: DatasetModel): Promise<DatasetModel[]> {
+        dataset: DatasetModel, pagination: PaginationModel):
+        Promise<any> {
+
         let query: any;
         if (dataset.gtags === undefined || dataset.gtags.length === 0) {
             query = journalClient.createQuery(
@@ -83,16 +85,27 @@ export class DatasetDAO {
             }
         }
 
-        let [entities] = await journalClient.runQuery(query);
+        if (pagination && pagination.cursor) { query = query.start(pagination.cursor); }
+
+        if (pagination && pagination.limit) { query = query.limit(pagination.limit); }
 
-        entities = (entities) as DatasetModel[];
+        const [entities, info] = await journalClient.runQuery(query);
 
         // Fix model for old entity
         for (let entity of entities) {
             entity = await this.fixOldModel(entity, dataset.tenant, dataset.subproject);
         }
 
-        return entities;
+        const output: PaginatedDatasetList = {
+            datasets: entities,
+            nextPageCursor: null
+        };
+
+        if (pagination) {
+            output.nextPageCursor = info.endCursor;
+        }
+
+        return output;
 
     }
 

src/services/dataset/handler.ts

@@ -318,7 +318,10 @@ export class DatasetHandler {
     private static async list(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
 
         // Retrieve the dataset path information
-        const dataset = DatasetParser.list(req);
+        const userInput = DatasetParser.list(req);
+
+        const dataset = userInput.dataset;
+        const pagination = userInput.pagination;
 
         // init journalClient client
         const journalClient = JournalFactoryTenantClient.get(tenant);
@@ -331,18 +334,21 @@ export class DatasetHandler {
                 req.headers['impersonation-token-context'] as string);
         }
 
-
         // Retrieve the list of datasets metadata
-        const datasets = await DatasetDAO.list(journalClient, dataset);
+        const output = await DatasetDAO.list(journalClient, dataset, pagination) as any;
 
         // attach the gcpid for fast check
-        for (const item of datasets) {
+        for (const item of output.datasets) {
             item.ctag = item.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);
         }
 
+
         // Retrieve the list of datasets metadata
-        return datasets;
+        if (output.nextPageCursor) {
+            return output;
+        }
 
+        return output.datasets;
     }
 
     // delete a dataset
@@ -834,7 +840,7 @@ export class DatasetHandler {
         // check if the dataset does not exist
         const lockKey = datasetIN.tenant + '/' + datasetIN.subproject + datasetIN.path + datasetIN.name;
         if (!dataset) {
-            if(await Locker.getLock(lockKey)) {
+            if (await Locker.getLock(lockKey)) {
                 // if a previous call fails, the dataset is not created but the lock is acquired and not released
                 await Locker.unlock(lockKey);
                 return;

src/services/dataset/model.ts

@@ -14,6 +14,7 @@
 // limitations under the License.
 // ============================================================================
 
+
 export interface IDatasetModel {
     name: string;
     tenant: string;
@@ -46,4 +47,9 @@ export interface IPaginationModel {
 export interface IDatasetAcl {
     admins: string[],
     viewers: string[];
+}
+
+export interface PaginatedDatasetList {
+    datasets: IDatasetModel[];
+    nextPageCursor: string;
 }
\ No newline at end of file

src/services/dataset/parser.ts

@@ -109,10 +109,28 @@ export class DatasetParser {
         req.query.seismicmeta === 'true'];
     }
 
-    public static list(req: expRequest): DatasetModel {
+    public static list(req: expRequest): any {
         const dataset = this.createDatasetModelFromRequest(req);
         dataset.gtags = req.query.gtag;
-        return dataset;
+
+        const limit = parseInt(req.query.limit, 10);
+        if (limit < 0) {
+            throw (Error.make(Error.Status.BAD_REQUEST,
+                'The \'limit\' query parameter can not be less than zero.'));
+        }
+        const cursor = req.query.cursor as string;
+        if (cursor === '') {
+            throw (Error.make(Error.Status.BAD_REQUEST,
+                'The \'cursor\' query parameter can not be empty if supplied'));
+        }
+        let pagination = null;
+        if (limit || cursor) {
+            pagination = { limit, cursor };
+        }
+
+        // Retrieve the list of datasets metadata
+        return { dataset, pagination };
+
     }
 
     public static delete(req: expRequest): DatasetModel {

src/services/imptoken/dao.ts

@@ -91,7 +91,11 @@ export class ImpTokenDAO {
         try {
             await request(options);
         } catch (error) {
-            throw (Error.make(Error.Status.BAD_REQUEST, 'The impersonation token cannot be refreshed.'));
+            // For any code different than 4xx the imptoken can be refreshed
+            // This is a temporary fix to handle unavailability of client infrastructure
+            if (error.statusCode >= 400 && error.statusCode <= 499) {
+                throw (Error.make(Error.Status.BAD_REQUEST, 'The impersonation token cannot be refreshed.'));
+            }
         }
 
     }

tests/utest/dao/dataset.ts

@@ -14,13 +14,12 @@
 // limitations under the License.
 // ============================================================================
 
-import sinon from 'sinon';
-
 import { Datastore } from '@google-cloud/datastore';
 import { Entity } from '@google-cloud/datastore/build/src/entity';
 import { RunQueryResponse } from '@google-cloud/datastore/build/src/query';
-import { google } from '../../../src/cloud/providers';
+import sinon from 'sinon';
 import { Config } from '../../../src/cloud';
+import { google } from '../../../src/cloud/providers';
 import { RecordLatency } from '../../../src/metrics';
 import { DatasetModel } from '../../../src/services/dataset';
 import { DatasetDAO } from '../../../src/services/dataset/dao';
@@ -28,6 +27,7 @@ import { Locker } from '../../../src/services/dataset/locker';
 import { IPaginationModel } from '../../../src/services/dataset/model';
 import { Tx } from '../utils';
 
+
 export class TestDataset {
 
 	private static dataset: DatasetModel;
@@ -82,13 +82,13 @@ export class TestDataset {
 
 		Tx.test(async (done: any) => {
 			this.journal.save.resolves({} as never);
-			await DatasetDAO.register(this.journal, { key: {'key': 'dataset_key'}, data: TestDataset.dataset });
+			await DatasetDAO.register(this.journal, { key: { 'key': 'dataset_key' }, data: TestDataset.dataset });
 			done();
 		});
 
 		Tx.test(async (done: any) => {
 			this.journal.save.resolves();
-			await DatasetDAO.register(this.journal, { key: {'key': 'dataset_key'}, data: TestDataset.dataset });
+			await DatasetDAO.register(this.journal, { key: { 'key': 'dataset_key' }, data: TestDataset.dataset });
 			done();
 		});
 	}
@@ -186,7 +186,7 @@ export class TestDataset {
 			this.journal.runQuery.resolves([expectedResult, undefined]);
 			this.sandbox.stub(DatasetDAO, 'fixOldModel').resolves(expectedResult[0]);
 
-			const result = await DatasetDAO.list(this.journal, this.dataset);
+			const result = await DatasetDAO.list(this.journal, this.dataset, null);
 
 			Tx.checkTrue(
 				this.journal.runQuery.calledWith(query) && result[0] === expectedResult[0],
@@ -231,7 +231,7 @@ export class TestDataset {
 			this.journal.runQuery.resolves([expectedResult, undefined]);
 			this.sandbox.stub(DatasetDAO, 'fixOldModel').resolves(expectedResult[0]);
 
-			const result = await DatasetDAO.list(this.journal, this.dataset);
+			const result = await DatasetDAO.list(this.journal, this.dataset, null);
 
 			Tx.checkTrue(
 				this.journal.runQuery.calledWith(query) && result[0] === expectedResult[0],
@@ -274,7 +274,7 @@ export class TestDataset {
 			this.journal.getQueryFilterSymbolContains.returns('=');
 			this.sandbox.stub(DatasetDAO, 'fixOldModel').resolves(expectedResult[0]);
 
-			const result = await DatasetDAO.list(this.journal, this.dataset);
+			const result = await DatasetDAO.list(this.journal, this.dataset, null);
 
 			Tx.checkTrue(
 				this.journal.runQuery.calledWith(query) && result[0] === expectedResult[0],