diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..f4ef881b9cf542860d8c92d2e90abccbb07d7eff --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,2 @@ +include: + - local: '/Reservoir DMS Suite/open-etp-server/.gitlab-ci.yml' \ No newline at end of file diff --git a/Reservoir DMS Suite/README.md b/Reservoir DMS Suite/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bb4809be2af550cf1493bf679110295b8b33eea5 --- /dev/null +++ b/Reservoir DMS Suite/README.md @@ -0,0 +1,14 @@ +# Reservoir Domain Data Management Service Suite + +The Reservoir Domain Data Management Service Suite (Reservoir-DDMS) is a group of services that provides storage for associated with seismic and well interpretation, structural modeling, geological modeling and reservoir modeling including reservoir simulation input. + +![Reservoir Domain Data Management Service Components](img/RDDMS-Components.svg) + +The main service [open-etp-server](open-etp-server/README.md), provides storage server on top of a POSTGRESQL compatible database, it is written in C++ and exposes the content using the Energistics Transfer Protocol API, allowing the efficient discovery and transfer of both metadata and binary data. Part of the server can also be used as as simple client to manipulate data partition, import and export RESQML files. + +Other services are participating in the integration of the main server with the OSDU ecosystem or providing alternative outside access: + +- A typescript client library (Being finalized), allows nodejs application to communicate with the server, and expose a REST API + + + diff --git a/Reservoir DMS Suite/img/RDDMS-Components.svg b/Reservoir DMS Suite/img/RDDMS-Components.svg new file mode 100644 index 0000000000000000000000000000000000000000..8546d101efafb3d2cbe11d817d132643fa8ed794 --- /dev/null +++ b/Reservoir DMS Suite/img/RDDMS-Components.svg @@ -0,0 +1 @@ +RESQMLFilesReservoir DDMSREST ClientEx: PostmanETP TypeScriptlibraryETP ClientRest ServerETP ServerC++ETP ClientC++, java, Python, Go ...PostgreSQL DBETPSQLRESTETPEPC+ HDF5 \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/.dockerignore b/Reservoir DMS Suite/open-etp-server/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..e741e6d59dc3900dc877497de80688c39a19407c --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/.dockerignore @@ -0,0 +1,8 @@ +# Exclude everything +** + +# Allow using dist for runtime image +!/build/using_docker/dist/** + +# allow using scripts for build +!/scripts/** diff --git a/Reservoir DMS Suite/open-etp-server/.gitignore b/Reservoir DMS Suite/open-etp-server/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..7b5ba340b295145336ff4b16508757d70983713a --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/.gitignore @@ -0,0 +1,14 @@ +build/** +Testing/Temporary/** + +*~ +*.swp +*.pyo +*.pyc + +# IDE folders +/.vscode/* +!/.vscode/tasks.json +!/.vscode/launch.json +/*.kdev4 +/.idea diff --git a/Reservoir DMS Suite/open-etp-server/.gitlab-ci.yml b/Reservoir DMS Suite/open-etp-server/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..9e2b071c48b1ed12eca650f33b6f9b8c4f49da5a --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/.gitlab-ci.yml @@ -0,0 +1,136 @@ +stages: + - build + - package + - test + - deploy + + +# .pre stage: running pipeline setup stuff +# Docker Image for build creation using docker-compose +DC-Build-stage: + stage: .pre + image: docker/compose:1.29.2 + tags: ['osdu-large'] + script: + - cd Reservoir\ DMS\ Suite/open-etp-server + - docker info + - echo "Running docker-compose build" + - docker-compose build + + +# Actually build OpenTPServer +OES-Build-stage: + stage: build + image: docker/compose:1.29.2 + tags: ['osdu-large'] + script: + - cd Reservoir\ DMS\ Suite/open-etp-server + - echo "Building binaries for OpenETPServer" + - echo "Give rights to posgres user to run init script:" + - chmod o+rwx -R scripts/init-db + - export DOCKER_GATEWAY_HOST=172.17.0.1 + - echo "Looking for host path for bind mounts" + - export CONTAINER_ID=$(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=$CI_JOB_ID" -f "label=com.gitlab.gitlab-runner.type=build") + - export MOUNT_NAME=$(docker inspect $CONTAINER_ID -f "{{ range .Mounts }}{{ if eq .Destination \"/builds\" }}{{ .Source }}{{end}}{{end}}") + - export SOURCE_HOST_DIR=$MOUNT_NAME/osdu/platform/domain-data-mgmt-services/reservoir/reservoir-ddms/Reservoir\ DMS\ Suite/open-etp-server + # --force recreate to ensure containers are new ones + # --renew-anon-volumes to ensure no previous data is used + # --exit-code-from to exit as soon as build is finished and use it exit code (failure reporting) + - docker-compose up --force-recreate --renew-anon-volumes --exit-code-from open_etp_server_build + cache: + # First cache for build speed up of all stuff! + key: build-cache + paths: + - Reservoir\ DMS\ Suite/open-etp-server/build/ + # Use cache from previous runs and push it to update it + policy: pull-push + # Generate a GitLab artifact of code coverage repots. + coverage: /^\s*lines:\s*\d+.\d+\%/ + artifacts: + name: openETPServer_coverage + reports: + cobertura: Reservoir\ DMS\ Suite/open-etp-server/build/using_docker_debug/gcovr/sonarqube/coverage.xml + paths: + - Reservoir\ DMS\ Suite/open-etp-server/build/using_docker_debug/gcovr/ + + +# Generate packages +OES-Packaging-stage: + stage: package + image: docker/compose:1.29.2 + tags: ['osdu-large'] + script: + - cd Reservoir\ DMS\ Suite/open-etp-server + - echo "Looking for host path for bind mounts" + - export CONTAINER_ID=$(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=$CI_JOB_ID" -f "label=com.gitlab.gitlab-runner.type=build") + - export MOUNT_NAME=$(docker inspect $CONTAINER_ID -f "{{ range .Mounts }}{{ if eq .Destination \"/builds\" }}{{ .Source }}{{end}}{{end}}") + - export SOURCE_HOST_DIR=$MOUNT_NAME/osdu/platform/domain-data-mgmt-services/reservoir/reservoir-ddms/Reservoir\ DMS\ Suite/open-etp-server + - echo "Create a runtime image" + - docker-compose build open_etp_server_runtime + cache: + key: build-cache + paths: + - Reservoir\ DMS\ Suite/open-etp-server/build/ + # Only pull cache, no need to update afterward + policy: pull + # Generate a GitLab artifact of the package. + artifacts: + name: openETPServer + paths: + - Reservoir\ DMS\ Suite/open-etp-server/build/using_docker/dist.tgz + + +# Run tests +OES-Tests-stage: + stage: test + image: docker/compose:1.29.2 + tags: ['osdu-large'] + script: + - cd Reservoir\ DMS\ Suite/open-etp-server + - echo "Looking for host path for bind mounts" + - export CONTAINER_ID=$(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=$CI_JOB_ID" -f "label=com.gitlab.gitlab-runner.type=build") + - export MOUNT_NAME=$(docker inspect $CONTAINER_ID -f "{{ range .Mounts }}{{ if eq .Destination \"/builds\" }}{{ .Source }}{{end}}{{end}}") + - export SOURCE_HOST_DIR=$MOUNT_NAME/osdu/platform/domain-data-mgmt-services/reservoir/reservoir-ddms/Reservoir\ DMS\ Suite/open-etp-server + - echo " --- See Build stage for Unit Tests ---" + - echo "Checking runtime image is running" + - docker-compose run open_etp_server_runtime + + +# Deploy +OES-Deployment-stage: + stage: deploy + image: docker/compose:1.29.2 + tags: ['osdu-large'] + script: + - cd Reservoir\ DMS\ Suite/open-etp-server + - echo "Looking for host path for bind mounts" + - export CONTAINER_ID=$(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=$CI_JOB_ID" -f "label=com.gitlab.gitlab-runner.type=build") + - export MOUNT_NAME=$(docker inspect $CONTAINER_ID -f "{{ range .Mounts }}{{ if eq .Destination \"/builds\" }}{{ .Source }}{{end}}{{end}}") + - export SOURCE_HOST_DIR=$MOUNT_NAME/osdu/platform/domain-data-mgmt-services/reservoir/reservoir-ddms/Reservoir\ DMS\ Suite/open-etp-server + - echo "Running deployment" + - echo " --- NOT YET IMPLEMENTED ---" + - echo "Listing files:" + - ls -al build/using_docker/dist + cache: + key: build-cache + paths: + - Reservoir\ DMS\ Suite/open-etp-server/build/ + # Only pull cache, no need to update afterward + policy: pull + + +# .post stage: running pipeline stopping stuff +# Docker Image for build creation using docker-compose +DC-Poweroff-stage: + stage: .post + image: docker/compose:1.29.2 + tags: ['osdu-large'] + script: + - cd Reservoir\ DMS\ Suite/open-etp-server + - echo "Looking for host path for bind mounts" + - export CONTAINER_ID=$(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=$CI_JOB_ID" -f "label=com.gitlab.gitlab-runner.type=build") + - export MOUNT_NAME=$(docker inspect $CONTAINER_ID -f "{{ range .Mounts }}{{ if eq .Destination \"/builds\" }}{{ .Source }}{{end}}{{end}}") + - export SOURCE_HOST_DIR=$MOUNT_NAME/osdu/platform/domain-data-mgmt-services/reservoir/reservoir-ddms/Reservoir\ DMS\ Suite/open-etp-server + - echo "Running docker-compose down" + - docker-compose down + allow_failure: true diff --git a/Reservoir DMS Suite/open-etp-server/.vscode/launch.json b/Reservoir DMS Suite/open-etp-server/.vscode/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..2e402d9a09ead193adcc4f3195fad8140d006e42 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/.vscode/launch.json @@ -0,0 +1,140 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "(gdb) Linux Launch", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/using_docker_debug/openETPServer", + "args": ["server","--start","-N","none","-M", "2GB"], + "stopAtEntry": true, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) Docker Launch", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/using_docker_debug/openETPServer", + "args": ["server","--start","-N","none","-M", "2GB"], + "stopAtEntry": true, + "cwd": "/source", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "sourceFileMap":{ + "/source":"${workspaceFolder}" + }, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + + { + "name": "(gdb) Docker Attach", + "type": "cppdbg", + "request": "attach", + "program": "${workspaceFolder}/build/using_docker/openETPServer", + "processId": "${command:pickProcess}", + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "sourceFileMap":{ + "/source":"${workspaceFolder}" + }, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) Launch Tests EML", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/using_docker_debug/tests/run_test_eml", + "args":[], + "stopAtEntry": true, + "cwd": "/source", + "environment": [ + {"name": "UNIT_TESTS_DATA_PATH","value": "${workspaceFolder}/build/using_docker_debug/tests"}, + ], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "sourceFileMap":{ + "/source":"${workspaceFolder}" + }, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) Launch Tests PostgreSQL", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/using_docker_debug/tests/run_test_postgresql", + "args":[], + "stopAtEntry": true, + "cwd": "/source", + "environment": [ + {"name": "UNIT_TESTS_DATA_PATH","value": "${workspaceFolder}/build/using_docker_debug/tests"}, + ], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "sourceFileMap":{ + "/source":"${workspaceFolder}" + }, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) Attach", + "type": "cppdbg", + "request": "attach", + "program": "${workspaceFolder}/build/using_docker/openETPServer", + "processId": "${command:pickProcess}", + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "sourceFileMap":{ + "/source":"${workspaceFolder}" + }, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + } + ] +} \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/.vscode/tasks.json b/Reservoir DMS Suite/open-etp-server/.vscode/tasks.json new file mode 100644 index 0000000000000000000000000000000000000000..be316fde56f079babbb34b0930becbaff9b821db --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/.vscode/tasks.json @@ -0,0 +1,51 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "Build release and debug using Docker", + "command": "bash", + "args": ["scripts/build_inside_docker.sh"], + "problemMatcher": { + "base": "$gcc", + "owner": "cpp", + "fileLocation": ["relative", "${workspaceRoot}"], + "pattern": { + "regexp": "^\\.\\.\/\\.\\.\/(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + }, + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "type": "shell", + "label": "Build debug in system", + "command": "bash", + "args": ["scripts/build.sh", "--debug", "build/using_docker_debug"], + "problemMatcher": { + "base": "$gcc", + "owner": "cpp", + "fileLocation": ["relative", "${workspaceRoot}"], + "pattern": { + "regexp": "^\\.\\.\/\\.\\.\/(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + }, + "group": { + "kind": "build", + "isDefault": true + } + } + ] + } \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/.gitignore b/Reservoir DMS Suite/open-etp-server/3rdParties/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..72e8ffc0db8aad71a934dd11e5968bd5109e54b4 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/3rdParties/.gitignore @@ -0,0 +1 @@ +* diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/AvroCPP/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/AvroCPP/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/Boost/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/Boost/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/Curl/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/Curl/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/HDF5/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/HDF5/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/Nlohmann/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/Nlohmann/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/OpenSSL/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/OpenSSL/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/PostgreSQL/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/PostgreSQL/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/3rdParties/WebSocketPP/.gitkeep b/Reservoir DMS Suite/open-etp-server/3rdParties/WebSocketPP/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Reservoir DMS Suite/open-etp-server/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..224356e6e6c4630de4a9fdf630569e9f7cb15ff8 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/CMakeLists.txt @@ -0,0 +1,144 @@ +# cmake_minimum_required() also set the newly-introduced policies to NEW +# See: https://cmake.org/cmake/help/latest/manual/cmake-policies.7.html +cmake_minimum_required(VERSION 3.15) + +if( POLICY CMP0074 ) + # New policy for detecting packages libraries was introduced in CMake 3.12 + # See https://cmake.org/cmake/help/v3.12/policy/CMP0074.html + cmake_policy(SET CMP0074 "NEW") +endif() + +if( POLICY CMP0077 ) + # New policy for option() behavior is to honor already existing cache entry + # See https://cmake.org/cmake/help/v3.13/policy/CMP0077.html + cmake_policy(SET CMP0077 "NEW") +endif() + +# Organize projects into folders +# See: http://athile.net/library/blog/?p=288 +set_property(GLOBAL PROPERTY USE_FOLDERS ON) + + +# Main Project for the whole repo +project( OpenETPServer ) + + +################################################################################ +# Options for tuning build rules + +# Location for custom 3rd parties: +set( OPEN_ETP_SERVER_3RDPARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/3rdParties/" CACHE PATH "Directory where to find 3rd parties libraries" ) +# Choose ABI policy +# see https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html +option( OLD_CXX11_ABI "Force old ABI policy for glibcxx" ) + +option( CODE_COVERAGE "Enable code coverage" ) + +################################################################################ +# Define C/CXX standards + +# Defines the C++ Standard to use (for GCC, equivalent to -std=c++17) +set( CMAKE_CXX_STANDARD 17 ) + +# Defines the C Standard to use (for GCC, equivalent to -std=c90 or -ansi) +set( CMAKE_C_STANDARD 90 ) + +set( CMAKE_CXX_STANDARD_REQUIRED ON ) +# CMake uses extended variants of language dialects by default, such as -std=gnu++14 instead of -std=c++14 +# Disable that. +set( CMAKE_CXX_EXTENSIONS OFF ) + + +################################################################################ +# Select the output directries for binaries + +# Generate all binaries at the root of the build tree: +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +set( RUNTIME_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + +# put libraries in a dedicated "lib" directory +if( WIN32 ) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + set( LIBRARY_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) + set( ARCHIVE_OUTPUT_PATH ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +else() + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) + set( LIBRARY_OUTPUT_PATH ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) +endif() + + +set( HELPER_LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake" CACHE INTERNAL "Where to find CMake helper scripts." FORCE ) +# Base definitions for 3rd parties: +include(${HELPER_LIBRARY_DIR}/3rd_parties.cmake) + + +################################################################################ +# OS/Compiler options: +if( UNIX ) + if( CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + add_definitions( + "-fdiagnostics-color=always" # To use color in messages with Ninja + "-pipe" # Use pipes instead of creating temporary files (improves performance) + "-Wall" "-Wextra" # Output warnings + ) + add_definitions( -fPIC ) + + if( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "7" AND OLD_CXX11_ABI) + add_definitions("-D_GLIBCXX_USE_CXX11_ABI=0") + endif() + + # Ensure GOLD linker is used + if( NOT CMAKE_SHARED_LINKER_FLAGS MATCHES "-fuse-ld=gold") + set( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold" ) + set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold" ) + endif() + + # Do not allow unresolved symbols for neither library nor executable. + if( NOT CMAKE_SHARED_LINKER_FLAGS MATCHES "--no-undefined") + set( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined" ) + set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined" ) + endif() + + # Ease GDB's life for faster debugging + if( NOT CMAKE_SHARED_LINKER_FLAGS MATCHES "--gdb-index") + set( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index" ) + set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index" ) + endif() + + # enable code covering + if( CODE_COVERAGE ) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage -fprofile-arcs -ftest-coverage") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -fprofile-arcs -ftest-coverage") + endif() + endif() +else() + add_compile_definitions(NOMINMAX _WIN32_WINNT=0x0601) +endif() + + +################################################################################ +# Enable testing via CTest for the project +enable_testing() + + +################################################################################ +# Custom definitions for project +add_definitions( + -DOES_EML_AUTH_NO_CRYPTO + # By commenting next line lots of "not supported in this build" issues in UT were resolved: + #-DOES_EML_ETP12_NO_AVRO_JSON + -DOES_CORE_UTILS_AVOID_BOOST +) + +IF(CMAKE_BUILD_TYPE MATCHES "Debug") + ADD_DEFINITIONS(-DOES_DEBUG_BUILD) +ENDIF() + +set( USE_POSTGRESQL_ETP ON ) +if(USE_POSTGRESQL_ETP) + ADD_DEFINITIONS(-DETP_POSTGRESQL) +endif() + +add_subdirectory(src) diff --git a/Reservoir DMS Suite/open-etp-server/Dockerfile.build b/Reservoir DMS Suite/open-etp-server/Dockerfile.build new file mode 100644 index 0000000000000000000000000000000000000000..6931e93cb3b1be96b6926457f5605d5db19db752 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/Dockerfile.build @@ -0,0 +1,82 @@ +FROM ubuntu:21.04 + +ARG VERSION=0.1 + +LABEL version="${VERSION}" +LABEL vendor="OSDU Reservoir DDMS" +LABEL description="Open ETP Server Building Image" + +VOLUME \ + # Source directory for OpenETPServer + /source + +ARG BOOST_VER="1.74*" +ARG CURL_VER="7.74*" +ARG POSTGRESQL_VER="13" + +RUN apt-get update && \ +# Define DEBIAN_FRONTEND to allow non interactive installation of some packages + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends \ + ca-certificates \ +# Build system packages + # build essential brings too much stuff + #build-essential=12.8* \ + # This subset is enough to build + libc6-dev=2.33* gcc=4:10.3* g++=4:10.3* \ + ninja-build=1.10* \ + libgomp1=11* \ + cmake=3.* \ + gdb \ +# 3rd parties + # For downloading AvroCPP package + wget \ + # For AvroCPP runtime + libboost-iostreams-dev=${BOOST_VER} \ + libboost-program-options-dev=${BOOST_VER} \ + libsnappy-dev=1.1* \ + \ + libboost-filesystem-dev=${BOOST_VER} \ + libboost-regex-dev=${BOOST_VER} \ + libboost-system-dev=${BOOST_VER} \ + libboost-thread-dev=${BOOST_VER} \ + libboost-test-dev=${BOOST_VER} \ + \ + libcurl4-openssl-dev=${CURL_VER} \ + \ +# doxygen=1.8* \ + \ + libhdf5-dev=1.10.6* \ + \ + nlohmann-json3-dev=3.9* \ + \ + libssl-dev=1.1.1* \ + openssl=1.1.1* \ + \ + libwebsocketpp-dev=0.8* \ + \ +# PostgreSQL required libraries + postgresql-server-dev-${POSTGRESQL_VER} \ + \ +# For debugging purposes +# strace=5.5* \ + \ +# For code coverage + gcovr \ + \ +# For the beauty of the output + figlet \ + \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Unfortunately Avrocpp is not available as is (avro-c is but not our need) +ARG AVROCPP_VER="1.10.2" +COPY scripts/build_and_install_avrocpp.sh /tmp/avro.sh +RUN /tmp/avro.sh && rm /tmp/avro.sh + +#################################################### + +WORKDIR /source +EXPOSE 9002 + +CMD [ "bash", "-c", "scripts/build_inside_docker.sh" ] diff --git a/Reservoir DMS Suite/open-etp-server/Dockerfile.runtime b/Reservoir DMS Suite/open-etp-server/Dockerfile.runtime new file mode 100644 index 0000000000000000000000000000000000000000..5609fd38d8c9dc8ad625a6dd0a7b72af25a81c2f --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/Dockerfile.runtime @@ -0,0 +1,13 @@ +FROM ubuntu:21.04 + +ARG VERSION=0.1 + +LABEL version="${VERSION}" +LABEL vendor="OSDU Reservoir DDMS" +LABEL description="Open ETP Server Runtime Image" + +COPY ./build/using_docker/dist/ /usr/ + +EXPOSE 9002 + +CMD [ "openETPServer", "server", "-h" ] \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/LICENSE b/Reservoir DMS Suite/open-etp-server/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..5266644a0b9f19dfd7ec746e38bec66c782d3937 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/LICENSE @@ -0,0 +1,232 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 Open Subsurface Data Universe Forum + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +========================================================== + + Use of Third Parties +---------------------------------------------------------- + FESAPI + +This software uses part of [FESAPI] (https://github.com/F2I-Consulting/fesapi), by F2I consulting which is available under Apache License, License 2.0. + +---------------------------------------------------------- + Minizip + +This software uses part of minizip by Gilles Vollant +Mini zip and unzip based on zlib +Includes Zip64 support by Mathias Svensson +See http://www.winimage.com/zLibDll/minizip.html + +---------------------------------------------------------- + MoodyCamel + +This software uses part of [MoodyCamel] (https://github.com/cameron314/concurrentqueue), which is available under a [Simplified BSD LIcense](https://github.com/cameron314/concurrentqueue/blob/master/LICENSE.md) + +---------------------------------------------------------- + SQLITE third party + +Some algorithms are based on [SQLITE] which is publicly (https://sqlite.org/copyright.html) available. + +---------------------------------------------------------- + C++ Formatting library + +This software use part of [Formatting library](https://github.com/fmtlib/fmt) for C++ by Victor Zverovich, publicly available under [License](https://github.com/fmtlib/fmt/blob/master/LICENSE.rst) \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/README.md b/Reservoir DMS Suite/open-etp-server/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bad10b15fbc6406c9a60607fa228ed67ad1bb281 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/README.md @@ -0,0 +1,180 @@ +# Reservoir DDMS - OpenETPServer + +OpenETPServer is the open source implementation of the Reservoir Domain Data Management Services (Reservoir-DDMS) which is one of the backend services part of Open Subsurface Data Universe (OSDU) software ecosystem. OpenETPServer is a single, containerized service written in C++ which stores reservoir data in RESQML format inside a PostgreSQL database. + +The service API is based on [Energistics Transfer Protocol version 1.2](https://public.3.basecamp.com/p/pNHktDcKVBGENgAbTu57YAeU), and allows the exchange of data based on [RESQML version 2.0.1+](http://docs.energistics.org/#RESQML/RESQML_TOPICS/RESQML-000-000-titlepage.html) format. + +## Build + +### Requirements + +- A Linux based system (or Windows Subsystem for Linux) +- Docker (which now comes with docker-compose) +- For older Docker versions: docker-compose + +### Build process + +The solution is based on two docker images: + +- open_etp_server_build is used to build the code both in debug and release, and run the unit tests +- open_etp_server_runtime is an image created with the result from open_etp_server_build and is packaging a server ready to run + +The solution is also using a postgres docker image, for the unit tests and in the default runtime mode. +To use an alternative postgres server you need to provide a postgres connection string in the variable POSTGRESQL_CONN_STRING with a format like "host=172.0.0.1 port=5432 dbname=postkv user=tester password=tester" + +### Creating Docker Image (Golden Path) + +To run docker container to build OpenETPServer and run unit tests: + +```bash +docker-compose up --abort-on-container-exit +``` + +Note that --abort-on-container-exit is used to remove the postgresql container required by unit tests + +To create and test the runtime image + +```bash +docker-compose up --abort-on-container-exit open_etp_server_runtime +``` + +This should display the help information about the ETP server options. + +In order to remove the running postgresql container: + +```bash +docker-compose down +``` + +### Using custom build + +If you prefer you can build inside a more conventional build environment. +To do so, simply create a build directory (for instance build/custom_build) and run +```bash +cmake SOURCE_DIRECTORY -GNinja +ninja +``` + +This can be adapted for your needs with normal CMake options. + +### Custom third-party libraries + +It is possible to use custom 3rd parties instead of system ones by filling the `3rdParties` directory. +It is also possible to keep 3rd parties inside a dedicated directory and provide it using the following option: +```bash +cmake SOURCE_DIR -GNinja -DOPEN_ETP_SERVER_3RDPARTY_DIR="PATH_TO_3RDPARTIES" +``` + +The provided directory must have the same structure than the `3rdParties` directory inside source tree. + + +## Usage + +### Starting an ETP API server + +The default configuration use the open_etp_postgresql container(see docker-compose.yml) from which the open_etp_server_build container depends, unless POSTGRESQL_CONN_STRING is redefining this behavior, see [Actual Build](#actualBuild). +The PostgreSQL "admin" schema is automatically created, checked or upgraded when the openETPServer is +starting. It is also possible to just do the initialization part using: + +```bash +# Initialize the database +docker-compose run open_etp_server_runtime openETPServer server --init +``` + +To start the etp server on its default port 9002, and using its default authentication mode: +(One user allowed with name=foo password=bar): + +```bash +# Start the ETP server +docker-compose run -p 9002:9002 open_etp_server_runtime openETPServer server --start + +# Server with default authentication with one credential: User: foo, Passwd: bar + +# Known issue: Since docker-compose does not wait for postgres to be ready, before starting the +# server, you may need to use this command twice + +# List all server options available +docker-compose up open_etp_server_runtime +``` + +### Interacting with the server data spaces + +The server executable is also implementing some client-side functionalities that allows content management. + +Once a container open_etp_server_runtime is running in server mode, we need to retrieve its name in order to fulfill the ETP server URL connection option (-S). +You can find its name using the following command. + +```bash +# Find the name of the container running the ETP server +docker container list + +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +0c66a13d35e9 open-etp-server_open_etp_server_build "/source/build/using…" About a minute ago Up About a minute 0.0.0.0:9002->9002/tcp, :::9002->9002/tcp open-etp-server_open_etp_server_build_run_6ccce8b7967e +``` + +The ETP server can be accessed on your local machine using the full container name: **ws://open-etp-server_open_etp_server_build_run_6ccce8b7967e:9002** + +Storage is organized in data spaces (See Etp 1.2 documentation), each data space defines a scope for unique identifiers of contained resources. +Consequently, two identical data can be ingested inside the solution without conflict when assigned to different data spaces. +Although ETP supports the concept of default data space, there is no default space in the RDDMS, and currently the RDDMS enforces that data space paths are composed of two parts, example 'project/study' + +The following script creates a new data space on the server, imports a RESQML file, checks its content and delete it. +Note that the EPC document and its HDF5 companion storing binary data must already be copied in the ./data directory so that it can be seen from inside the container. + +```bash +# Create a new data space named 'demo/Volve' +docker-compose run open_etp_server_runtime openETPServer space -S ws://open-etp-server_open_etp_server_build_run_6ccce8b7967e:9002 -u foo -p bar --new -s demo/Volve + +# Make a list of all data spaces on the server +docker-compose run open_etp_server_runtime openETPServer space -S ws://open-etp-server_open_etp_server_build_run_6ccce8b7967e:9002 -u foo -p bar -l + +# Import a RESQML file in the Volve data space +# This assumes that the EPC and HDF5 companion are available in ./data directory +docker-compose run open_etp_server_build openETPServer space -S ws://open-etp-server_open_etp_server_build_run_6ccce8b7967e:9002 -u foo -p bar -s demo/Volve --import-epc /data/Volve_Demo_Reservoir_Grid_Depth.epc + +# Check the content +docker-compose run open_etp_server_build openETPServer space -S ws://open-etp-server_open_etp_server_build_run_6ccce8b7967e:9002 -u foo -p bar -s demo/Volve --stats + +# Delete an existing data space named 'demo/Volve' +docker-compose run open_etp_server_build openETPServer space --delete -S ws://open-etp-server_open_etp_server_build_run_6ccce8b7967e:9002 -s demo/Volve +``` + +## Additional Notes for developers + +## Developing inside the build container using VSCode + +Required Visual Studio extension: + +- C++: ms-vscode.cpptools +- Remote container support: ms-vscode-remote.remote-containers + +Create the container image using the instruction in [Actual Build](#actualBuild) section + +In order to debug you can use: + +```bash +docker-compose run open_etp_server_build bash +``` + +It will launch a batch in the build container, allowing to interact with it, or attach VSCode to the container. + +Open the open-etp-server directory in Visual Studio Code(View->Command Palette Ctrl+P). Then choose: Remote container/Attach to running container... +Sometimes you may need to reopen the folder in /source +The first time in the container, you may have to (re)enabled the C++ extension locally. +This create a new Visual Studio Code window that you can use to build (Ctrl B) and to debug (Ctrl D) + +### Selecting PostgreSQL Server + +docker-compose will also start a container of a PostgreSQL instance. + +Running: + +```bash +docker-compose run open_etp_server_build bash +``` + +will use the docker postgres image so that PostgreSQL dependent unit test can run. + +It is also possible to work with a pre-existing PostgreSQL server and not using the container. +You simply need to update the POSTGRESQL_CONN_STRING in the launch.json for the Unit tests and +in the docker-compose.yml for the server usage. diff --git a/Reservoir DMS Suite/open-etp-server/azure-pipelines.yml b/Reservoir DMS Suite/open-etp-server/azure-pipelines.yml new file mode 100644 index 0000000000000000000000000000000000000000..3192238358c9ad03d6a3e3a3db87816f4593277c --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/azure-pipelines.yml @@ -0,0 +1,118 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +name: Fy-$(Build.DefinitionName)-$(Build.SourceBranchName)-$(date:yyyyMMdd)$(rev:.r) + +trigger: + - main + - master + +variables: + - name: ForceClean + # Must define a pipeline variable named FORCE_CLEAN exposed to user for behavior modification + value: $(FORCE_CLEAN) + + - name: isMain + value: ${{ or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }} + + - name: Node.Version + value: 14.x + + # defines secret vars + - group: general-build-vars + + # Define Registry Variables + - name: RegistryURL + value: "portalvsts.azurecr.io" + - name: RegistryServiceConnection + value: "portalvsts" + + - name: RepositoryName + value: open-etp-server + + # Images names: + - name: ComposeServiceNameBuild + value: "open_etp_server_build" + - name: ComposeServiceNameRuntime + value: "open_etp_server_runtime" + +pool: + name: pdgmDockerLinux + # Temporarilly set to agent 4 for stability + demands: Agent.Name -equals af-ncy-ub20-4 + +steps: +## Do not clean the workspace unless user chose it. +- checkout: self + clean: $(ForceClean) +## Clean workspace EXCEPT build directory +- script: | + git clean -fdx -e build/ + +## Build Image +- task: DockerCompose@0 + displayName: "Build openETPServer build image" + inputs: + containerregistrytype: "Container Registry" + dockerRegistryEndpoint: "$(RegistryServiceConnection)" + dockerComposeFile: "docker-compose.yml" + action: "Run a Docker Compose command" + dockerComposeCommand: "build" + arguments: "$(ComposeServiceNameBuild)" +- task: DockerCompose@0 + displayName: "Compile openETPServer executable" + inputs: + containerregistrytype: "Container Registry" + dockerRegistryEndpoint: "$(RegistryServiceConnection)" + dockerComposeFile: "docker-compose.yml" + action: "Run a Docker Compose command" + dockerComposeCommand: "up" + arguments: "$(ComposeServiceNameBuild)" + +## Runtime Image +- task: DockerCompose@0 + displayName: "Build openETPServer runtime image" + inputs: + containerregistrytype: "Container Registry" + dockerRegistryEndpoint: "$(RegistryServiceConnection)" + dockerComposeFile: "docker-compose.yml" + action: "Run a Docker Compose command" + dockerComposeCommand: "build" + arguments: "$(ComposeServiceNameRuntime)" +- task: DockerCompose@0 + displayName: "Verify openETPServer runtime image" + inputs: + containerregistrytype: "Container Registry" + dockerRegistryEndpoint: "$(RegistryServiceConnection)" + dockerComposeFile: "docker-compose.yml" + action: "Run a Docker Compose command" + dockerComposeCommand: "run" + arguments: "$(ComposeServiceNameRuntime)" + +- task: Docker@2 + displayName: "Tag openETPServer runtime image: $(Build.BuildNumber)" + inputs: + command: "tag" + arguments: "$(RegistryURL)/$(ComposeServiceNameRuntime) $(RegistryURL)/$(RepositoryName):$(Build.BuildNumber)" + +- task: Docker@2 + displayName: "Tag openETPServer runtime image: latest" + inputs: + command: "tag" + arguments: "$(RegistryURL)/$(ComposeServiceNameRuntime) $(RegistryURL)/$(RepositoryName):latest" + condition: eq(variables.isMain, true) + +- task: Docker@2 + displayName: "Push openETPServer runtime image" + inputs: + containerRegistry: "$(RegistryServiceConnection)" + repository: "$(RepositoryName)" + command: "push" + ${{ if eq(variables.isMain, true) }}: + tags: | + $(Build.BuildNumber) + latest + ${{ if eq(variables.isMain, false) }}: + tags: $(Build.BuildNumber) diff --git a/Reservoir DMS Suite/open-etp-server/cmake/3rd_parties.cmake b/Reservoir DMS Suite/open-etp-server/cmake/3rd_parties.cmake new file mode 100644 index 0000000000000000000000000000000000000000..aa4cded75c6745a2c8e84e84bc4a448392d72cab --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/cmake/3rd_parties.cmake @@ -0,0 +1,311 @@ + +# 3rd parties: +# Comment following line for getting more information when 3rd party is searched +set( THIRDPARTY_VERBOSITY QUIET ) + +# 3rd parties minimal version required: +set( MINIMAL_AVROCPP_VERSION 1.9.2 ) +set( MINIMAL_BOOST_VERSION 1.71.0 ) +set( MINIMAL_CURL_VERSION 7.68.0 ) +set( MINIMAL_HDF5_VERSION 1.10.4 ) +set( MINIMAL_OPENSSL_VERSION 1.0.2 ) +set( MINIMAL_POSTGRESQL_VERSION 12.7 ) +set( MINIMAL_ZLIB_VERSION 1.2.11 ) + +# Enable Open MP in code. +macro( USE_OPENMP ) + # Allow OpenMP usage: + find_package(OpenMP REQUIRED) + if (OPENMP_FOUND) + if( NOT "${CMAKE_C_FLAGS}" MATCHES "${OpenMP_C_FLAGS}" ) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + endif() + if( NOT "${CMAKE_CXX_FLAGS}" MATCHES "${OpenMP_CXX_FLAGS}" ) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + endif() + + # Uniformize 3rd parties variables + set(OPENMP_LIBRARIES "${OpenMP_CXX_LIBRARIES}") + endif() +endmacro() + + +# Macro for using AvroCPP inside a binary +macro( USE_AVROCPP ) + # AvroCPP is not part of standard linux paquages. + # It needs to be build and installed. + + # Search AvroCPP includes on system: + find_path(AVROCPP_INCLUDE_DIRS + NAMES + avro/Encoder.hh + HINTS # Use 'HINTS' instead of 'PATHS' to precede system search + "${OPEN_ETP_SERVER_3RDPARTY_DIR}/AvroCPP/include" + ) + if( NOT AVROCPP_INCLUDE_DIRS ) + message( FATAL_ERROR + "AvroCPP include directory not found!" + " Try to build and install it," + " or use a pre-build version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/AvroCPP" + ) + endif() + + # Add include library for headers resolving + include_directories(SYSTEM "${AVROCPP_INCLUDE_DIRS}" ) + + # Search libs in the system + find_library(AVROCPP_LIBRARIES + NAMES + avrocpp + HINTS # Use 'HINTS' instead of 'PATHS' to precede system search + "${OPEN_ETP_SERVER_3RDPARTY_DIR}/AvroCPP/lib" # Local toolkits + ) + if( NOT AVROCPP_LIBRARIES ) + message( FATAL_ERROR + "AvroCPP libraries not found!" + " Try to build and install it," + " or use a pre build version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/AvroCPP" + ) + endif() +endmacro() + + +# Macro for using BOOST inside a binary +macro( USE_BOOST ) + if( IS_DIRECTORY "${OPEN_ETP_SERVER_3RDPARTY_DIR}/Boost/include" ) + # From Boost 1.70.0, there can be some cases where boost detection conflicts + # due to a cmake-built boost on the system. + # So we disable the default Boost CMake detection script. + # See https://cmake.org/cmake/help/v3.16/module/FindBoost.html#boost-cmake + set(Boost_NO_BOOST_CMAKE TRUE) + + set(BOOST_ROOT "${OPEN_ETP_SERVER_3RDPARTY_DIR}/Boost") + set(Boost_NO_SYSTEM_PATH TRUE) + else() + unset(Boost_NO_BOOST_CMAKE) + unset(BOOST_ROOT) + set(Boost_NO_SYSTEM_PATH FALSE) + endif() + + set(BOOST_MODULES date_time filesystem iostreams program_options regex system thread unit_test_framework) + if( WIN32 ) + list(APPEND BOOST_MODULES random ) + endif() + find_package( + Boost ${MINIMAL_BOOST_VERSION} + COMPONENTS + ${BOOST_MODULES} + REQUIRED + ${THIRDPARTY_VERBOSITY} + ) + + if( NOT Boost_FOUND ) + message( FATAL_ERROR + "Boost libraries not found!" + " Try to build and install it," + " or use a pre build version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/Boost" + ) + endif() + + # Add include library for headers resolving + include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ) + + # Uniformize 3rd parties variables + set(BOOST_LIBRARIES ${Boost_LIBRARIES}) + + # Extra Windows specific compilation flags + if(WIN32) + # Boost.Process 1.71+ fails without windows.h. + # WIN32_LEAN_AND_MEAN avoids issues with asio. + # https://github.com/boostorg/process/issues/96 + add_compile_definitions(BOOST_ALL_NO_LIB BOOST_ALL_DYN_LINK BOOST_USE_WINDOWS_H WIN32_LEAN_AND_MEAN) + endif() +endmacro() + + +# Macro for using cURL inside a binary +macro( USE_CURL ) + if( IS_DIRECTORY "${OPEN_ETP_SERVER_3RDPARTY_DIR}/Curl/include" ) + # Use Curl from local 3rd parties: + set( CURL_INCLUDE_DIRS "${OPEN_ETP_SERVER_3RDPARTY_DIR}/Curl/include") + if( UNIX ) + set( CURL_LIBRARIES curl) + else() + set( CURL_LIBRARIES libcurl) + endif() + link_directories( "${OPEN_ETP_SERVER_3RDPARTY_DIR}/Curl/lib" ) + else() + find_package( + CURL ${MINIMAL_CURL_VERSION} + REQUIRED + ${THIRDPARTY_VERBOSITY} + ) + + if( NOT CURL_FOUND ) + message ( FATAL_ERROR "cURL not found!" ) + endif() + endif() + + # Add include library for headers resolving + include_directories(SYSTEM "${CURL_INCLUDE_DIRS}" ) +endmacro() + + +# Macro for using HDF5 inside a binary +macro( USE_HDF5 ) + if( IS_DIRECTORY "${OPEN_ETP_SERVER_3RDPARTY_DIR}/HDF5/include" ) + set(HDF5_ROOT "${OPEN_ETP_SERVER_3RDPARTY_DIR}/HDF5") + set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE true) + + set(HDF5_INCLUDE_DIRS "${OPEN_ETP_SERVER_3RDPARTY_DIR}/HDF5/include") + set( HDF5_LIBRARIES + hdf5_cpp + hdf5 + hdf5_hl + ) + if( UNIX ) + set( HDF5_LIBRARIES ${HDF5_LIBRARIES} "szip") + else() + set( HDF5_LIBRARIES ${HDF5_LIBRARIES} "zlib") + endif() + link_directories("${OPEN_ETP_SERVER_3RDPARTY_DIR}/HDF5/lib/") + else() + unset(HDF5_ROOT) + unset(HDF5_NO_FIND_PACKAGE_CONFIG_FILE) + find_package( + HDF5 ${MINIMAL_HDF5_VERSION} + COMPONENTS + CXX C HL + ${THIRDPARTY_VERBOSITY} + ) + + if( NOT HDF5_FOUND ) + message( FATAL_ERROR + "HDF5 library not found!" + " Try to build and install it," + " or use a pre build version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/HDF5" + ) + endif() + + endif() + + # Add include library for headers resolving + include_directories(SYSTEM "${HDF5_INCLUDE_DIRS}" ) + + # As default is to use static libraries, + # force HDF5 t be dynamically loaded + add_definitions(-DH5_BUILT_AS_DYNAMIC_LIB) + +endmacro() + + +macro( USE_NLOHMANN ) + find_path(NLOHMANN_INCLUDE_DIRS + NAMES + nlohmann/json.hpp + HINTS # Use 'HINTS' instead of 'PATHS' to precede system search + "${OPEN_ETP_SERVER_3RDPARTY_DIR}/Nlohmann/include" + ) + + if( NOT NLOHMANN_INCLUDE_DIRS ) + message( FATAL_ERROR + "Nlohmann library not found!" + " Try to install it," + " or use a custom version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/Nlohmann" + ) + endif() + + # Add include library for headers resolving + include_directories(SYSTEM "${NLOHMANN_INCLUDE_DIRS}" ) + + # No .so for Header only library + set(NLOHMANN_LIBRARIES "") +endmacro() + + +# Macro for using OpenSSL inside a binary +macro( USE_OPENSSL ) + if( IS_DIRECTORY "${OPEN_ETP_SERVER_3RDPARTY_DIR}/OpenSSL/include" ) + set(OPENSSL_ROOT_DIR "${OPEN_ETP_SERVER_3RDPARTY_DIR}/OpenSSL") + else() + unset(OPENSSL_ROOT_DIR) + endif() + + # Find OpenSSL on system: + find_package( + OpenSSL ${MINIMAL_OPENSSL_VERSION} + REQUIRED + ${THIRDPARTY_VERBOSITY} + ) + + if( NOT OPENSSL_FOUND ) + message( FATAL_ERROR + "OpenSSL library not found!" + " Try to build and install it," + " or use a pre build version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/OpenSSL" + ) + endif() + + # Add include library for headers resolving + include_directories(SYSTEM "${OPENSSL_INCLUDE_DIR}" ) +endmacro() + + +# Macro for using PostgreSQL inside a binary +macro( USE_POSTGRESQL ) + if( IS_DIRECTORY "${OPEN_ETP_SERVER_3RDPARTY_DIR}/PostgreSQL/include" ) + # Use PostgreSQL from local 3rd parties: + set( PostgreSQL_INCLUDE_DIRS "${OPEN_ETP_SERVER_3RDPARTY_DIR}/PostgreSQL/include") + if( UNIX ) + set( PostgreSQL_LIBRARIES pq pgtypes) + else() + set( PostgreSQL_LIBRARIES libpq libpgtypes) + endif() + link_directories( "${OPEN_ETP_SERVER_3RDPARTY_DIR}/PostgreSQL/lib" ) + else() + # Find PostgreSQL on system: + find_package( + PostgreSQL ${MINIMAL_POSTGRESQL_VERSION} + REQUIRED + ${THIRDPARTY_VERBOSITY} + ) + + if( NOT PostgreSQL_FOUND ) + message( FATAL_ERROR + "PostgreSQL libraries not found!" + " Try to build and install it," + " or use a pre build version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/PostgreSQL" + ) + endif() + endif() + + # Add include library for headers resolving + include_directories(SYSTEM "${PostgreSQL_INCLUDE_DIRS}" ) + + # Uniformize 3rd parties variables + set(POSTGRESQL_LIBRARIES "${PostgreSQL_LIBRARIES}") +endmacro() + + +macro( USE_WEBSOCKETPP ) + find_path(WEBSOCKETPP_INCLUDE_DIRS + NAMES + websocketpp/version.hpp + HINTS # Use 'HINTS' instead of 'PATHS' to precede system search + "${OPEN_ETP_SERVER_3RDPARTY_DIR}/WebSocketPP/include" + ) + + if( NOT WEBSOCKETPP_INCLUDE_DIRS ) + message( FATAL_ERROR + "WebSocketPP libraries not found!" + " Try to install it," + " or use a custom version in ${OPEN_ETP_SERVER_3RDPARTY_DIR}/WebSocketPP" + ) + endif() + + # Add include library for headers resolving + include_directories(SYSTEM "${WEBSOCKETPP_INCLUDE_DIRS}" ) + + # No .so for Header only library + set(WEBSOCKETPP_LIBRARIES "") +endmacro() \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/cmake/embed_resource.cmake b/Reservoir DMS Suite/open-etp-server/cmake/embed_resource.cmake new file mode 100644 index 0000000000000000000000000000000000000000..e2ba65a128b30f0caa75f3948cba9f2a83f6003a --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/cmake/embed_resource.cmake @@ -0,0 +1,34 @@ +#################################################################################################### +# This function converts any file into C/C++ source code. +# Example: +# - input file: data.dat +# - output file: data.h +# - variable name declared in output file: DATA +# - data length: sizeof(DATA) +# embed_resource("data.dat" "data.h" "DATA") +#################################################################################################### + +function(embed_resource resource_file_name source_file_name variable_name) + + set( output_file "${CMAKE_CURRENT_BINARY_DIR}/${source_file_name}") + set(input_file "${CMAKE_CURRENT_SOURCE_DIR}/${resource_file_name}") + + # Only generate file if destination does not exists or is older than input file + if( "${input_file}" IS_NEWER_THAN "${output_file}" ) + file(READ ${input_file} hex_content HEX) + + string(REPEAT "[0-9a-f]" 32 column_pattern) + string(REGEX REPLACE "(${column_pattern})" "\\1\n" content "${hex_content}") + + string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1, " content "${content}") + + string(REGEX REPLACE ", $" "" content "${content}") + + set(array_definition "static const unsigned char ${variable_name}[] =\n{\n${content}\n};") + + set(source "// Auto generated file.\n${array_definition}\n") + + file(WRITE "${output_file}" "${source}") + endif() + +endfunction() diff --git a/Reservoir DMS Suite/open-etp-server/cmake/extract_dependencies.cmake.in b/Reservoir DMS Suite/open-etp-server/cmake/extract_dependencies.cmake.in new file mode 100644 index 0000000000000000000000000000000000000000..766791337224520f9e3bacbe10e02ce5f9d916d3 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/cmake/extract_dependencies.cmake.in @@ -0,0 +1,20 @@ + +message("Recursively install libraries for run time.") +execute_process( + COMMAND "@CMAKE_SOURCE_DIR@/scripts/extract_dependencies.py" + --include-system-libs + --outdir "." + --quiet + "bin/@PROJECT_NAME@" + WORKING_DIRECTORY @CMAKE_INSTALL_PREFIX@ + RESULT_VARIABLE extract_dependencies_result + #COMMAND_ECHO STDOUT + OUTPUT_VARIABLE extract_dependencies_stdout + ERROR_VARIABLE extract_dependencies_stdout +) + +message("${extract_dependencies_stdout}") + +if( extract_dependencies_result ) + message( FATAL_ERROR "Command failed with error: '${extract_dependencies_result}'") +endif() diff --git a/Reservoir DMS Suite/open-etp-server/create_demo_space.sh b/Reservoir DMS Suite/open-etp-server/create_demo_space.sh new file mode 100644 index 0000000000000000000000000000000000000000..cfda05ece7110e01807a05ba796d10fc082748d4 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/create_demo_space.sh @@ -0,0 +1,8 @@ +#!/bin/bash +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar --new -s demo/Volve +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./Volve_Demo_Fault_Horizon_TIME.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./Volve_Demo_Horizons_Depth.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./Volve_Demo_Reservoir_Grid_Depth.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./Volve_Demo_Wells_Depth.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --stats +# openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --delete diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Fault_Horizon_TIME.epc b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Fault_Horizon_TIME.epc new file mode 100644 index 0000000000000000000000000000000000000000..35fe99473b72eba5d030766d23c983d96c66e3bc Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Fault_Horizon_TIME.epc differ diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Fault_Horizon_TIME.h5 b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Fault_Horizon_TIME.h5 new file mode 100644 index 0000000000000000000000000000000000000000..e39c4abc30c0d9322bc3f2abcd76928e5842b814 Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Fault_Horizon_TIME.h5 differ diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Horizons_Depth.epc b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Horizons_Depth.epc new file mode 100644 index 0000000000000000000000000000000000000000..4d78c06d22ab435199245ae4ccd385e86bca01da Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Horizons_Depth.epc differ diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Horizons_Depth.h5 b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Horizons_Depth.h5 new file mode 100644 index 0000000000000000000000000000000000000000..91531c8f5fc588c038ffdf393a519063f5868962 Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Horizons_Depth.h5 differ diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Reservoir_Grid_Depth.epc b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Reservoir_Grid_Depth.epc new file mode 100644 index 0000000000000000000000000000000000000000..286e8df5221cf1a67cd7a6ef1a10ca624088b26f Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Reservoir_Grid_Depth.epc differ diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Reservoir_Grid_Depth.h5 b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Reservoir_Grid_Depth.h5 new file mode 100644 index 0000000000000000000000000000000000000000..51bae8d49bad4db1fd2920aaefa18926e9917357 Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Reservoir_Grid_Depth.h5 differ diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Wells_Depth.epc b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Wells_Depth.epc new file mode 100644 index 0000000000000000000000000000000000000000..9f9a5e062c28c2e387ffd1c10fda036c5fd7be6b Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Wells_Depth.epc differ diff --git a/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Wells_Depth.h5 b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Wells_Depth.h5 new file mode 100644 index 0000000000000000000000000000000000000000..c6712ea3a171c2755139a6887d9d47a79a46f5f8 Binary files /dev/null and b/Reservoir DMS Suite/open-etp-server/data/Volve_Demo_Wells_Depth.h5 differ diff --git a/Reservoir DMS Suite/open-etp-server/data/create_demo_space.sh b/Reservoir DMS Suite/open-etp-server/data/create_demo_space.sh new file mode 100755 index 0000000000000000000000000000000000000000..d6040862f5fdc875949966133952d86b85d0995e --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/data/create_demo_space.sh @@ -0,0 +1,8 @@ +export SERVER_CONTAINER=open-etp-server_open_etp_server_runtime_run_5981f91fc84f +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar --new -s demo/Volve +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./data/Volve_Demo_Fault_Horizon_TIME.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./data/Volve_Demo_Horizons_Depth.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./data/Volve_Demo_Reservoir_Grid_Depth.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --import-epc ./data/Volve_Demo_Wells_Depth.epc +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --stats +openETPServer space -S ws://$SERVER_CONTAINER:9002 -u foo -p bar -s demo/Volve --delete diff --git a/Reservoir DMS Suite/open-etp-server/docker-compose.yml b/Reservoir DMS Suite/open-etp-server/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..55863fc51ac3f3b5bed8d3cfc16ed6925e3ce45b --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/docker-compose.yml @@ -0,0 +1,62 @@ +version: '3' + +services: + open-etp-postgresql: + image: postgres + container_name: open-etp-postgresql + ports: + - "5432:5432" + environment: + - POSTGRES_PASSWORD=tester + - POSTGRES_USER=tester + - POSTGRES_DB=postkv + volumes: + - ${SOURCE_HOST_DIR:-.}/scripts/init-db:/docker-entrypoint-initdb.d + networks: + - web + + open_etp_server_build: + build: + context: . + dockerfile: Dockerfile.build + args: + - VERSION=0.1 + image: open_etp_server_build + labels: + OpenETPServer.service: build + logging: + driver: json-file + depends_on: + - open-etp-postgresql + environment: + - POSTGRESQL_CONN_STRING=host=open-etp-postgresql port=5432 dbname=postkv user=tester password=tester + volumes: + - ${SOURCE_HOST_DIR:-.}:/source + networks: + - web + + open_etp_server_runtime: + profiles: ["cli-only"] + depends_on: + - open-etp-postgresql + build: + context: . + dockerfile: Dockerfile.runtime + args: + - VERSION=0.1 + image: open_etp_server_runtime + volumes: + - ${SOURCE_HOST_DIR:-.}/data:/data + ports: + - "9002:9002" + labels: + OpenETPServer.service: runtime + logging: + driver: json-file + environment: + - POSTGRESQL_CONN_STRING=${POSTGRESQL_CONN_STRING:-host=open-etp-postgresql port=5432 dbname=postkv user=tester password=tester} + networks: + - web + +networks: + web: diff --git a/Reservoir DMS Suite/open-etp-server/list_unused_files.py b/Reservoir DMS Suite/open-etp-server/list_unused_files.py new file mode 100644 index 0000000000000000000000000000000000000000..c31922fc445bbb3bf86cb1416f4d29fbb33198bf --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/list_unused_files.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import pathlib +import re +import os + + +ignored_files_re = re.compile(r"(CMakeLists.txt|files.cmake)$") +files_cmake_re = re.compile(r"(.*\.(?:h|hpp|hh|c|cc|cpp))$", re.IGNORECASE) + + +def handle_files_cmake(files_cmake): + print(f"handling {files_cmake}") + files_cmake_dir = files_cmake.parent + used_files = set() + with open(files_cmake, "rt") as fd: + for line in fd: + # cleanup line from comments + line = line.split("#")[0].strip() + if m:=files_cmake_re.match(line): # noqa: E231 E701 + used_files.add(files_cmake_dir / m.group(1)) + # list all files in dir and subdirs: + full_list = set() + for f in files_cmake_dir.rglob("*"): + if f.is_dir(): + continue + elif ignored_files_re.match(str(f.name)): + continue + elif files_cmake_re.match(str(f)): + full_list.add(f) + else: + # Uncomment next line for detecting unhandeled files + # print(" ", f) + pass + unused_files = full_list - used_files + if unused_files: + print(" Files CMake is not aware of:") + for f in sorted(unused_files): + print(f" {f.relative_to(os.getcwd())}") + print(f" Files used: {len(used_files)}/{len(full_list)}") + else: + print(f" All {len(full_list)} files are well used") + return True + return False + + +def main(): + root_dir = pathlib.Path(__file__).parent.absolute() + print(root_dir) + result = True + for item in sorted(root_dir.rglob("files.cmake")): + result = handle_files_cmake(item) and result + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/Reservoir DMS Suite/open-etp-server/scripts/build.sh b/Reservoir DMS Suite/open-etp-server/scripts/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..e33c2da7f01f42f70b34b8650184ede9dbf8fd11 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/scripts/build.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Set value to 0 to force PostgreSQL unit tests execution +NO_POSTGRESQL_UNIT_TESTS=1 + +if [[ $# -gt 0 && "$1" == "--debug" ]]; then + BUILD_TYPE=Debug + CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCODE_COVERAGE=ON" + # remove the parameter + shift +else + BUILD_TYPE=Release + CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}" +fi + +if [[ $# -lt 1 ]]; then + echo -e "$0 [--debug] BUILD_DIR [ninja_args]..." + echo -e "\--debug\n\t\tPerform Debug build (Default: Release build)" + echo -e "\tBUILD_DIR\n\t\tPath to the build directory to use" + echo -e "\tninja_args\n\t\tAny argument that will be forwarded to ninja" + exit 1 +fi + +# Coloration codes +Black='\033[0;30m' +DarkGray='\033[1;30m' +Red='\033[0;31m' +LightRed='\033[1;31m' +Green='\033[0;32m' +LightGreen='\033[1;32m' +Orange='\033[0;33m' +Yellow='\033[1;33m' +Blue='\033[0;34m' +LightBlue='\033[1;34m' +Purple='\033[0;35m' +LightPurple='\033[1;35m' +Cyan='\033[0;36m' +LightCyan='\033[1;36m' +LightGray='\033[0;37m' +White='\033[1;37m' +NC='\033[0m' # No Color + +# Adding error reporting +err_report() { + echo "" + echo -e "${Red}Error on $0:$1${NC}: stop execution." +} +trap 'err_report $LINENO' ERR + + +# Actual build +SOURCE_TREE="${PWD}" +BUILD_TREE="${SOURCE_TREE}/${1}" + +# Removing first mandatory parameter +shift + +# To know the global execution time: +main_start=`date +%s` + +if [ ! -d "$BUILD_TREE" ]; then + echo "Creating build directory" + mkdir -p "${BUILD_TREE}" && echo -e "\tOK" +fi + +cd "${BUILD_TREE}" + + +echo -e "${LightCyan}-------------------------------------------------------------------------${NC}" +echo -e "${LightBlue}Cleanup UnitTest directory${NC}" +rm -rf tests + +echo -e "${LightBlue}Cleanup Install directory${NC}" +rm -rf dist + +echo -e "${LightBlue}Cleanup CodeCoverage directory${NC}" +rm -rf gcovr + +echo -e "${LightCyan}-------------------------------------------------------------------------${NC}" +echo -e "${LightBlue}Configuring build tree using CMake and Ninja for ${BUILD_TYPE} build${NC}" + +echo -e "Running (in ${Green}$PWD${NC}): ${Yellow}cmake -GNinja ${CMAKE_OPTIONS} \"${SOURCE_TREE}\" -DCMAKE_INSTALL_PREFIX=\"./dist\"${NC}" +start=`date +%s` +cmake -GNinja ${CMAKE_OPTIONS} "${SOURCE_TREE}" -DCMAKE_INSTALL_PREFIX="./dist" +end=`date +%s` +echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" + + +# Actual build: +echo -e "${LightCyan}-------------------------------------------------------------------------${NC}" +echo -e "${LightBlue}Building${NC}" +echo -e "Running (in ${Green}$PWD${NC}): ${Yellow}ninja $*${NC}" +start=`date +%s` +ninja "$@" +end=`date +%s` +echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" + +# Remove local temporary file +rm -f a.out + + +echo -e "${LightCyan}-------------------------------------------------------------------------${NC}" +for filename in openETPServer; do + if [[ -f ${filename} && -x ${filename} ]]; then + echo -e "${LightBlue}Testing that ${filename} can be run${NC}" + echo -e "Running (in ${Green}$PWD${NC}) (log written in ${Cyan}${filename}.log${NC}): ${Yellow}./${filename} -h${NC}" + start=`date +%s` + if (./${filename} -h > ./${filename}.log 2>&1); then + end=`date +%s` + echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" + else + echo -e "\t${Red}FAILED with return code: $?${NC}" + echo -e "${Purple}<<<<< STDOUT <<<<<${NC}" + cat ./${filename}.log + echo -e "${Purple}>>>>> STDOUT >>>>>${NC}" + exit 1 + fi + fi +done + + +echo -e "${LightCyan}-------------------------------------------------------------------------${NC}" +echo -e "${LightBlue}Running UnitTests:${NC}" +echo -e "Running (in ${Green}$PWD${NC}) (log written in ${Cyan}${PWD}/Testing/Temporary/LastTest.log${NC}): ${Yellow}ctest . -v -j4${NC}" +start=`date +%s` +if (ctest . -v -j4); then + end=`date +%s` + echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" +else + echo -e "\t${Red}FAILED with return code: $?${NC}" + echo -e "${Purple}<<<<< logging file <<<<<${NC}" + cat ./Testing/Temporary/LastTest.log + echo -e "${Purple}>>>>> logging file >>>>>${NC}" + echo -e "You can read this content in file: ${Yellow}${PWD}/Testing/Temporary/LastTest.log${NC}" + exit 1 +fi + + +echo -e "${LightCyan}-------------------------------------------------------------------------${NC}" +if [ ${BUILD_TYPE} == "Debug" ]; then + echo -e "${LightBlue}Computing Code Coverage reports (can take some time):${NC}" + echo -e "Running (in ${Green}$PWD${NC}): ${Yellow}gcovr -r ../.. -j 8 -s --delete (...) -o gcovr.log .${NC}" + start=`date +%s` + mkdir -p gcovr/html + mkdir -p gcovr/sonarqube + if (gcovr -r ../.. -j 8 -s --delete --html "gcovr/html/index.html" --html-details --sonarqube "gcovr/sonarqube/openetpserver.xml" -o "gcovr.log" .); then + end=`date +%s` + echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" + else + echo -e "\t${Red}FAILED with return code: $?${NC}" + echo -e "${Purple}<<<<< logging file <<<<<${NC}" + cat ./gcovr.log + echo -e "${Purple}>>>>> logging file >>>>>${NC}" + echo -e "You can read this content in file: ${Yellow}${PWD}/Testing/Temporary/LastTest.log${NC}" + # Cleanup GCOV files if not done by gcov + find "${SOURCE_TREE}" -name *.gcov -exec rm -r {} \; + exit 1 + fi +else + echo -e "${LightBlue}Packaging:${NC}" + echo -e "Running (in ${Green}$PWD${NC}) (log written in ${Cyan}${filename}.log${NC}): ${Yellow}ninja install${NC}" + start=`date +%s` + ninja install + end=`date +%s` + echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" + + echo -e "\n${LightBlue}Checking dist program can start:${NC}" + echo -e "Running (in ${Green}$PWD${NC}) (log written in ${Cyan}${PWD}/start_dist.log${NC}): ${Yellow}./dist/bin/openETPServer -h${NC}" + start=`date +%s` + if (LD_LIBRARY_PATH=./dist/lib ./dist/bin/openETPServer -h > start_dist.log 2>&1); then + end=`date +%s` + echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" + else + echo -e "\t${Red}FAILED with return code: $?${NC}" + echo -e "${Purple}<<<<< logging file <<<<<${NC}" + cat ./start_dist.log + echo -e "${Purple}>>>>> logging file >>>>>${NC}" + echo -e "You can read this content in file: ${Yellow}${PWD}/Testing/Temporary/LastTest.log${NC}" + exit 1 + fi + + echo -e "\n${LightBlue}Compressing dist program:${NC}" + cd dist + echo -e "Running (in ${Green}$PWD${NC}) (log written in ${Cyan}${PWD}/../compress_dist.log${NC}): ${Yellow}tar -czvf ../dist.tgz .${NC}" + start=`date +%s` + if (tar -czvf ../dist.tgz .> ../compress_dist.log 2>&1); then + end=`date +%s` + echo -e "\t${Green}OK (`expr $end - $start`s)${NC}" + else + echo -e "\t${Red}FAILED with return code: $?${NC}" + echo -e "${Purple}<<<<< logging file <<<<<${NC}" + cat ../compress_dist.log + echo -e "${Purple}>>>>> logging file >>>>>${NC}" + echo -e "You can read this content in file: ${Yellow}${PWD}/Testing/Temporary/LastTest.log${NC}" + exit 1 + fi + cd .. +fi + +echo -e "${LightCyan}-------------------------------------------------------------------------${NC}" +end=`date +%s` +echo -e "Build and tests are ${Green}OK (`expr $end - $main_start`s)${NC}" diff --git a/Reservoir DMS Suite/open-etp-server/scripts/build_and_install_avrocpp.sh b/Reservoir DMS Suite/open-etp-server/scripts/build_and_install_avrocpp.sh new file mode 100755 index 0000000000000000000000000000000000000000..80d15515300d75a60de636abe8addaa00c910a63 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/scripts/build_and_install_avrocpp.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Note: AVROCPP_VER comes from Docker environment when built using Docker/DockerCompose +echo "Building and Installing AvroCPP version ${AVROCPP_VER}" + +avro_cpp_build_dir=/tmp/avro_build + +mkdir -p "${avro_cpp_build_dir}" +cd "${avro_cpp_build_dir}" +wget https://github.com/apache/avro/archive/refs/tags/release-${AVROCPP_VER}.tar.gz -O - | tar -xz + +mkdir -p "${avro_cpp_build_dir}/avro-release-${AVROCPP_VER}/lang/c++/build" +cd "${avro_cpp_build_dir}/avro-release-${AVROCPP_VER}/lang/c++/build" + +cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. + +ninja install + +rm -rf "${avro_cpp_build_dir}" diff --git a/Reservoir DMS Suite/open-etp-server/scripts/build_inside_docker.sh b/Reservoir DMS Suite/open-etp-server/scripts/build_inside_docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..41cc603cec38ebc19c53390b013c08d6a3a92ad8 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/scripts/build_inside_docker.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Create group and user identically to running user. +if [ ! "$(getent group running_user_group)" ]; then + echo "Creating running user group" + SOURCE_TREE_GID=$(stat -c '%g' .) + if [ "$SOURCE_TREE_GID" -eq "0" ]; then + SOURCE_TREE_GID=12345 + fi + groupadd -r running_user_group -g "${SOURCE_TREE_GID}" && echo -e "\tOK" +fi +if [ ! "$(getent passwd running_user)" ]; then + echo "Creating running user" + SOURCE_TREE_UID=$(stat -c '%u' .) + if [ "$SOURCE_TREE_UID" -eq "0" ]; then + SOURCE_TREE_UID=12345 + fi + useradd --no-log-init -r -u "${SOURCE_TREE_UID}" -g running_user_group running_user && echo -e "\tOK" +fi + +echo "#########################################################################" +figlet -c "Debug" +su -m running_user -c "bash scripts/build.sh --debug build/using_docker_debug" + +echo "#########################################################################" +figlet -c "Release" +su -m running_user -c "bash scripts/build.sh build/using_docker" diff --git a/Reservoir DMS Suite/open-etp-server/scripts/extract_dependencies.py b/Reservoir DMS Suite/open-etp-server/scripts/extract_dependencies.py new file mode 100755 index 0000000000000000000000000000000000000000..ddcce2546703cf341d4df011d7eb06008d194720 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/scripts/extract_dependencies.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- + +import subprocess +import re +import pathlib +import os +import sys +import argparse +import shutil +import textwrap + + +DEBUG = 0 +QUIET = 0 +LISTED_FILES = set() + + +def parse_cli() -> argparse.Namespace: + """Reads command arguments. + + Returns: + argparse.Namespace: options given by user in command line. + """ + global DEBUG, QUIET + parser = argparse.ArgumentParser( + description=textwrap.dedent(""" + Given a binary, lists all its dependencies, optionally copy it to the given output directory. + It is recommended to ran `setvars.sh` first to get needed environment variables. + """).strip() + ) + parser.add_argument( + "FILES", nargs="+", + help="list of binaries to get/copy dependencies from", + ) + parser.add_argument( + "-d", "--debug", + dest="debug", action="store_true", + help="Print more information about what is done", + ) + parser.add_argument( + "-q", "--quiet", + dest="quiet", action="store_true", + help="Print as few info as possible", + ) + parser.add_argument( + "-s", "--system", "--include-system-libs", + dest="include_system_libs", action="store_true", + help="Copy system libraries to destination.", + ) + parser.add_argument( + "-o", "--outdir", + dest="output_directory", default=None, action="store", + help="Directory where to put shared libs and binaries to (if not given, lists only)", + ) + options = parser.parse_args() + DEBUG = DEBUG or options.debug + QUIET = options.quiet + return options + + +def is_lib(item: pathlib.Path) -> bool: + """Detects if a file is a shared library based on filename parsing. + + Args: + item (pathlib.Path): item to check. + + Returns: + bool: True if `item` is a shared library. + """ + return ".so" in f".{item.name.split('.', 1)[-1]}" + + +def handle_file(item: pathlib.Path, base_dest: pathlib.Path = None) -> None: + """Prints and eventually copies given `item`. + + Args: + item (pathlib.Path): Item to print and copy. + base_dest (pathlib.Path): Optional directory where to copy `item`. Defaults to None. + """ + if base_dest: + # Handle Copy + if is_lib(item): + dest = base_dest / "lib" / item.name + else: + dest = base_dest / "bin" / item.name + dest.parent.mkdir(parents=True, exist_ok=True) + if DEBUG: + print(f" Copy '{item}' to '{dest}'") + elif not QUIET: + print(f" --> Installing: {dest.absolute()}") + if item.absolute() == dest.absolute(): + return + if item.is_symlink(): + target = (item.parent / os.readlink(item)).absolute() + try: + target = target.relative_to(item.parent) + except ValueError: + pass # Not relative, copy it as is + dest.symlink_to(target=target) + else: + shutil.copy2(item, dest) + else: + if DEBUG: + print(item.is_symlink(), item) + else: + print(item) + + +def handle_all_links(item: pathlib.Path, base_dest: pathlib.Path = None) -> None: + """Given `item` file, prints and eventually copies it and its symlinks targets. + + Args: + item (pathlib.Path): Item to handle (if it is a symlink, then will handle target too). + base_dest (pathlib.Path): Optional directory where to copy `item`. Defaults to None. + """ + global LISTED_FILES + if item not in LISTED_FILES: + handle_file(item, base_dest) + LISTED_FILES.add(item) + else: + # if item is already listed all link are too. + return + + # If symlinks, handle target too. + if item.is_symlink(): + handle_all_links((item.parent / os.readlink(item)).absolute(), base_dest) + + +def print_all_dependencies(item: pathlib.Path, base_dest: pathlib.Path = None, include_system_libs: bool = False) -> int: + """Given a binary (executable or library) runs `ldd` to list all its dependencies. + It eventually copies all items if `base_dest` is given. + It uses both OES_PROJECT_BUILD_DIR and OES_TOOLKITS_DIR environment variables + to detect dependencies. + + Args: + item (pathlib.Path): Path to the binary to analyze. + base_dest (pathlib.Path): Optional directory where to copy `item`. Defaults to None. + include_system_libs (bool): Copy system libraries too. Defaults to False. + + Returns: + int: 0 if all was OK; 1 if anything went wrong. + """ + base_dir = pathlib.Path( + os.environ.get( + "OES_PROJECT_BUILD_DIR", + os.getcwd() # Default for current working directory + ) + ).absolute() + + home_toolworks = pathlib.Path( + os.environ.get( + "OES_TOOLKITS_DIR", + "/home/toolworks" # default to /home/toolworks + ) + ).absolute() + + if DEBUG: + print(dir(base_dir)) + + possible_files = ( + pathlib.Path(item), + base_dir / item, + base_dir / "bin" / item, + base_dir / "lib" / item, + ) + for app_file in possible_files: + if app_file.is_file(): + break + if not app_file.is_file(): + print(f"Error: cannot find file '{item}'") + return 1 + + # Input must be part of the package too. + handle_all_links(app_file, base_dest) + + output = subprocess.check_output( + f'ldd {app_file}', + stderr=subprocess.STDOUT, + shell=True + ).decode() + + found_re = re.compile(r"\s*(.*)\s=>\s(.+)\s\(.*\)") + not_found_re = re.compile(r"\s*(.*)\s=>\snot\sfound") + + for line in output.splitlines(): + if m := not_found_re.search(line): + print("Not found", m.group(1)) + return 1 + if m := found_re.search(line): + lib_path = pathlib.Path(m.group(2)) + lib_path_abs = lib_path.absolute() + common_paths = { + os.path.commonpath((base_dir, lib_path_abs)), + os.path.commonpath((home_toolworks, lib_path_abs)) + } + if common_paths == {"/"} and not include_system_libs: + continue + handle_all_links(lib_path_abs, base_dest) + + return 0 + + +def main() -> int: + """Parses CLI options and list/copy all dependencies to user's options. + + Returns: + int: 0 if all was OK; 1 if anything went wrong. + """ + options = parse_cli() + if options.output_directory: + base_path = pathlib.Path(options.output_directory) + base_path.mkdir(parents=True, exist_ok=True) + else: + base_path = None + + os.environ["LD_LIBRARY_PATH"] = ":".join([os.path.join(".", "lib"), "/usr/local/lib"] + os.environ.get("LD_LIBRARY_PATH", [])) + for item in options.FILES: + if print_all_dependencies(item, base_path, options.include_system_libs): + return 1 + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/Reservoir DMS Suite/open-etp-server/scripts/init-db/init.sh b/Reservoir DMS Suite/open-etp-server/scripts/init-db/init.sh new file mode 100644 index 0000000000000000000000000000000000000000..945ae3f9db25a895015cb6f5a5f2f600ef7048fe --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/scripts/init-db/init.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE USER UTEST; + CREATE EXTENSION lo; +EOSQL + + diff --git a/Reservoir DMS Suite/open-etp-server/src/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1fe71d0ab6ff33b48bd787ccd436d7a8266f17af --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/CMakeLists.txt @@ -0,0 +1,7 @@ + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib) + +add_subdirectory(lib) +add_subdirectory(bin) + +add_subdirectory(tests) diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/bin/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a93282215c90fb27cc9e27e6c6133bf79dc313c3 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(openETPServer) diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..3eb732e773374687527f53f2442d589f07fa4ccd --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/CMakeLists.txt @@ -0,0 +1,75 @@ +project(openETPServer) + +# 3rd parties: + USE_AVROCPP() + USE_BOOST() + USE_HDF5() + if(USE_POSTGRESQL_ETP) + USE_POSTGRESQL() + endif() + USE_WEBSOCKETPP() + + +# files needed for build + include(files.cmake) + +add_executable( + openETPServer + ${openETPServer_HEADERS} + ${openETPServer_SOURCES} +) + +if(USE_POSTGRESQL_ETP) + target_link_libraries( + openETPServer + # Internal library that are explicitelly inside header files + PUBLIC + + # Internal library that are only in source files + PRIVATE + oes_utils + oes_http + oes_eml + oes_postgresql + # 3rd Parties: + PRIVATE + ${AVROCPP_LIBRARIES} + ${BOOST_LIBRARIES} + ${HDF5_LIBRARIES} + ${WEBSOCKETPP_LIBRARIES} + ${POSTGRESQL_LIBRARIES} + ) +else() + target_link_libraries( + openETPServer + # Internal library that are explicitelly inside header files + PUBLIC + + # Internal library that are only in source files + PRIVATE + oes_utils + oes_http + oes_eml + # 3rd Parties: + PRIVATE + ${AVROCPP_LIBRARIES} + ${BOOST_LIBRARIES} + ${HDF5_LIBRARIES} + ${WEBSOCKETPP_LIBRARIES} + ) +endif() + +if(USE_POSTGRESQL_ETP) + install( + TARGETS openETPServer oes_utils oes_http oes_eml oes_postgresql + ) +else() + install( + TARGETS openETPServer oes_utils oes_http oes_eml + ) +endif() + +configure_file( "${HELPER_LIBRARY_DIR}/extract_dependencies.cmake.in" "extract_dependencies.cmake" @ONLY) +install( + SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/extract_dependencies.cmake" +) diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EmlAppContext.cpp b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EmlAppContext.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dc92a3467411ba0fb8d4fd6c50cc20d4d5c4ec2a --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EmlAppContext.cpp @@ -0,0 +1,461 @@ +// ============================================================================ +// Copyright 2015-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "EmlAppContext.h" + +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +# include +# include +# define isatty(h) _isatty(h) +#else +#include +#include +#include +#endif + +namespace po = boost::program_options; +using oes::apps::eml::EmlAppContext; + +using std::endl; + +namespace { + +// http://stackoverflow.com/questions/6240950 +std::ostream cnull(nullptr); + +// See also http://stackoverflow.com/questions/16935908 +// and http://stackoverflow.com/questions/6812224 +std::pair getTerminalLineAndColumnSizes() { + std::pair result{ 0, 0 }; + +#ifdef _WIN32 + CONSOLE_SCREEN_BUFFER_INFO csbi; + if (0 != GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi)) { + result.first = csbi.srWindow.Bottom - csbi.srWindow.Top + 1; + result.second = csbi.srWindow.Right - csbi.srWindow.Left + 1; + } +#else + struct winsize w; + if (-1 != ioctl(STDOUT_FILENO, TIOCGWINSZ, &w)) { + result.first = w.ws_row; + result.second = w.ws_col; + } +#endif + + if (result.first == 0) { + result.first = 24; + } else { + result.first = std::min(std::max(result.first, 4u), 256u); // clamp( 4, 256) + } + + if (result.second == 0) { + result.second = po::options_description::m_default_line_length; + } else { + // Call succeeded. But values may be out of range. + // Redirected output on Linux yields (6, 0) for example. + result.second = std::min(std::max(result.second, 64u), 160u); // clamp(64, 160) + } + + return result; +} + +} // namespace + +// FIXME: lots of copy/paste + +namespace oes { +namespace apps { +namespace eml { + +/****************************************************************************/ + +#ifdef OES_DEBUG_BUILD +bool test_parseByteSize() { + auto assertOK = [](size_t expected, const char* spec) { + assert(expected == parseByteSize(spec)); + }; + assertOK(1, "1"); + assertOK(1, "1B"); + assertOK(1000, "1KB"); + assertOK(1024, "1KiB"); + assertOK(15*1000*1000, "15MB"); + assertOK(size_t(1.5*1000*1000), "1.5MB"); + assertOK(size_t(1.5*1024*1024), "1.5MiB"); + + auto assertKO = [](const char* expected_err_fragment, const char* spec) { + try { + parseByteSize(spec); + assert(false); + } catch (const std::runtime_error& ex) { + const std::string err(ex.what()); + assert(err.find(expected_err_fragment) != std::string::npos); + } + }; + assertKO("Expected leading number", "foo"); + assertKO("Must end with B", "1K"); + assertKO("Invalid uom", "1FooB"); + assertKO("Invalid uom", "1FB"); + assertKO("Size too large", "3GB"); + assertKO("Size too large", "1TB"); + + return true; +} +static const bool ok = test_parseByteSize(); +#endif + +size_t parseByteSize(const std::string& spec) { + std::stringstream ss(spec); + + double val = 0; + ss >> val; + if (!ss) { + throw std::runtime_error("Expected leading number in " + spec); + } + + if (val < 0 || val > 2147483647 /*2GiB*/) { + throw std::runtime_error(fmt::format("Unexpected size: {}", val)); + } + + std::string uom; + ss >> uom; + + if (uom.empty()) { + // No unit, assume bytes directly + return static_cast(val); + } + + if (uom.back() != 'B') { + throw std::runtime_error(fmt::format("Invalid uom: {}: Must end with B, for Byte", uom)); + } + + const bool power_of_2 = (uom.size() == 3 && uom[1] == 'i'); + + if (!(uom.size() == 1 || uom.size() == 2 || power_of_2)) { + throw std::runtime_error(fmt::format("Invalid uom: {}. Use KB, MB, GB; KiB, MiB, GiB", uom)); + } + + const double factor = power_of_2? 1024: 1000; + + switch (uom[0]) { + case 'T': val *= factor; BOOST_FALLTHROUGH; + case 'G': val *= factor; BOOST_FALLTHROUGH; + case 'M': val *= factor; BOOST_FALLTHROUGH; + case 'K': val *= factor; BOOST_FALLTHROUGH; + case 'B': break; + default: + throw std::runtime_error(fmt::format("Invalid uom: {}. Use KB, MB, GB; KiB, MiB, GiB", uom)); + } + + if (val > 2147483647 /*2GiB*/) { + throw std::runtime_error(fmt::format("Size too large (>2GiB): {}", val)); + } + + return static_cast(val); +} + +std::pair parseByteSizes(const std::string& spec) { + std::vector sizes; + if (spec.empty()) { + return { 0, 0 }; + } + + boost::split( + sizes, spec, boost::algorithm::is_any_of("\t ,;|"), + boost::algorithm::token_compress_on + ); + + size_t zsize = 0; // max compressed size + size_t usize = 0; // max uncompressed size + + switch (sizes.size()) { + case 1: + zsize = parseByteSize(sizes[0]); + usize = 10 * zsize; // default to 10x the specified max-compressed-size + break; + case 2: + zsize = parseByteSize(sizes[0]); + usize = parseByteSize(sizes[1]); + break; + default: + throw std::runtime_error( + fmt::format("Expected 1 or 2 sizes, not {}", sizes.size()) + ); + } + + return { zsize, usize }; +} + +/****************************************************************************/ + +EmlAppContext::EmlAppContext( + variables_map& args +) : args(args) + , help(args.count("help") > 0) + , quiet(args.count("quiet") > 0) // quiet "wins" over verbose and debug + , debug((!quiet && args.count("debug") > 0)) // debug implies verbose too + , verbose((!quiet && (debug || args.count("verbose") > 0))) + , timer(args.count("timer") > 0) + , command(!args["command"].empty()? args["command"].as(): std::string()) + , terminalWidth_(terminalWidth()) +{ +} + +std::ostream& EmlAppContext::out() const { + const std::string& filename = stringArg("output-file"); + if (!filename.empty()) { + if (!ofs_.is_open()) { + ofs_.open(filename.c_str()); + } + if (ofs_.good()) { + return ofs_; + } + err() << "openETPServer: Warning: Cannot open file: " << filename << endl; + // fall thru + } + return quiet? cnull: std::cout; +} + +std::ostream& EmlAppContext::err() const { + return quiet? cnull: std::cerr; +} + +std::ostream& EmlAppContext::msg() const { + return verbose? out() : cnull; +} + +std::ostream& EmlAppContext::dbg() const { + return debug ? out() : cnull; +} + +bool EmlAppContext::hasArg(const std::string& name) const { + return args.count(name) > 0; +} + +std::string EmlAppContext::stringArg(const std::string& name) const { + const po::variable_value& value = args[name]; + return value.empty()? std::string(): value.as(); +} + +/*! + * \brief Augments global options with command-specific options. + * + * \param cmd_desc the command-specific options. + * \param unparsed_args the unparsed arguments to parse against \a cmd_desc. + * \return 0 on success; -1 on success, but return immediately success (0); + * positive error code otherwise. + */ +int EmlAppContext::reparse( + const options_description& cmd_desc, + const std::vector& unparsed_args +) { + try { + po::store( + po::command_line_parser(unparsed_args) + .options(cmd_desc) + .run(), + args + ); + } + catch (const std::exception& ex) { + err() << "openETPServer: Error: " << ex.what() + << '\n' << '\n' << cmd_desc << endl; + return 1; // error + } + + if (args.size() == 0) { + out() << cmd_desc << endl; + return -1; // success, but caller should return 0 + } + + if (help) { + out() << cmd_desc << endl; + return -1; // success, but caller should return 0 + } + + return 0; // success +} + +/*! + * \brief Augments global options with command-specific options. + * + * \param cmd_desc the command-specific options. + * \param pos_desc the command-specific positional options (option with no name) + * \param unparsed_args the unparsed arguments to parse against \a cmd_desc. + * \return 0 on success; -1 on success, but return immediately success (0); + * positive error code otherwise. + */ +int EmlAppContext::reparse( + const options_description& cmd_desc, + const positional_options_description& pos_desc, + const std::vector& unparsed_args +) { + try { + po::store( + po::command_line_parser(unparsed_args) + .options(cmd_desc) + .positional(pos_desc) + .run(), + args + ); + } + catch (const std::exception& ex) { + err() << "openETPServer: Error: " << ex.what() + << '\n' << '\n' << cmd_desc << endl; + return 1; // error + } + + if (args.size() == 0) { + out() << cmd_desc << endl; + return -1; // success, but caller should return 0 + } + + if (help) { + out() << cmd_desc << endl; + return -1; // success, but caller should return 0 + } + + return 0; // success +} + +int EmlAppContext::dispatchCommand( + const parsed_options& parsed, const EmlAppNamedCommandVec& commands +) { + OES_ASSERT(!commands.empty(), return 1;); + + for (const EmlAppNamedCommand& entry : commands) { + if (command == entry.name) { + std::vector command_args = po::collect_unrecognized( + parsed.options, po::include_positional + ); + + // Remove the command itself from command_args. + // Note that other unknown options can precede it. + const size_t before_size = command_args.size(); + OES_ASSERT(before_size >= 1, /*do nothing*/;); + for (size_t i = 0; i < command_args.size(); ++i) { + if (command_args[i] == command) { + command_args.erase(command_args.begin() + i); + break; + } + } + + return entry.command(*this, command_args); + } + } + + return -1; // no match. Unknown command? +} + +int EmlAppContext::dispatchAction(const EmlAppNamedAction actions[], size_t action_count) { + OES_ASSERT(actions && action_count, return 1;); + + size_t pop_count = 0; + const EmlAppNamedAction* p_action = nullptr; + for (size_t i = 0; i < action_count; ++i) { + const EmlAppNamedAction& action = actions[i]; + if (hasArg(action.name)) { + ++pop_count; + p_action = &action; + } + } + switch (pop_count) { + case 0: + err() << "Error: No action specified." << std::endl; + return 1; + case 1: + break; + default: + err() << "Error: Too many actions specified." << std::endl; + return 2; + } + + assert(p_action); + return timeCall(*p_action); +} + +const std::string& EmlAppContext::action() const { + OES_ASSERT(!action_.empty(), ;); + return action_; +} + +int EmlAppContext::timeCall(const EmlAppNamedAction& entry) { + const double user_start = oes::core::System::getUserTime(); + const double real_start = oes::core::System::getRealTime(); + + const int rc = tryCall(entry); + + if (timer && !quiet) { + const double user = oes::core::System::getUserTime() - user_start; + const double real = oes::core::System::getRealTime() - real_start; + oes::core::Format fmt = oes::core::Format::tr("%1|.3f|s (user: %2|.3f|s)"); + fmt % real % user; + std::string timing; + fmt >> timing; + out() << '\n' << command << (entry.name? " ": "") << (entry.name? entry.name: "") + << " in " << timing << ' ' + << oes::core::System::getMemoryUsage().peakMB() << " peak memory." + << endl; + } + + return rc; +} + +int EmlAppContext::tryCall(const EmlAppNamedAction& entry) { + try { + if (entry.name) { + action_ = entry.name; + } else { // validate and compare commands have no action + action_ = command; + } + return entry.action(*this)? 0: 1; + } catch (ActionFailed) { + return 1; + } catch (const std::exception& ex) { + err() << "openETPServer: Error: " << ex.what(); + return 3; // keep in sync with main() + } catch (...) { + err() << "openETPServer: Unknown Error" << endl; + return 4; // keep in sync with main() + } +} + +unsigned EmlAppContext::terminalWidth() { + return getTerminalLineAndColumnSizes().second; +} + +/****************************************************************************/ + +} // namespace eml +} // namespace apps +} // namespace oes diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EmlAppContext.h b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EmlAppContext.h new file mode 100644 index 0000000000000000000000000000000000000000..1d23248d22eeeac29c9d99f6d6ade0fdf96207e1 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EmlAppContext.h @@ -0,0 +1,167 @@ +// ============================================================================ +// Copyright 2015-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#pragma once +#include +#include +#include + +#include +#include +#include +#include + +namespace oes::apps::eml { + +using boost::program_options::options_description; +using boost::program_options::positional_options_description; +using boost::program_options::parsed_options; +using boost::program_options::variables_map; + +/****************************************************************************/ + +struct EmlAppContext; +using UnparsedArgs = std::vector; + +using EmlAppCommand = int (*)(EmlAppContext&, const UnparsedArgs&); +struct EmlAppNamedCommand { const char* name; EmlAppCommand command; const char* desc; }; +using EmlAppNamedCommandVec = std::vector; + +using EmlAppAction = bool (*)(const EmlAppContext&); +struct EmlAppNamedAction { const char* name; EmlAppAction action; }; + +struct ActionFailed {}; // Error message already printed, just "abort" current command + +/****************************************************************************/ + +struct EmlAppContext { + variables_map& args; + + const bool help; + const bool quiet; // warning: order matters + const bool debug; + const bool verbose; + const bool timer; + + const std::string command; + + const unsigned terminalWidth_; + +public: + EmlAppContext(variables_map& args); + + std::ostream& out() const; + std::ostream& err() const; + std::ostream& msg() const; + std::ostream& dbg() const; + + bool hasArg(const std::string& name) const; + std::string stringArg(const std::string& name) const; + + const std::string& action() const; + + int reparse( + const options_description& cmd_desc, + const std::vector& unparsed_args + ); + int reparse( + const options_description& cmd_desc, + const positional_options_description& pos_desc, + const std::vector& unparsed_args + ); + + int dispatchCommand( + const parsed_options& parsed, const EmlAppNamedCommandVec& commands + ); + int dispatchAction(const EmlAppNamedAction actions[], size_t action_count); + int timeCall(const EmlAppNamedAction& action); + int tryCall(const EmlAppNamedAction& action); + + template + int dispatchActions(const EmlAppNamedAction (&actions)[N]) { + return dispatchAction(actions, N); + } + + static unsigned terminalWidth(); + +private: // no copy/assignment + EmlAppContext(const EmlAppContext&) = delete; + EmlAppContext& operator=(const EmlAppContext&) = delete; + +private: + std::string action_; + mutable std::ofstream ofs_; +}; + +/****************************************************************************/ + +int hdf5_main (EmlAppContext& ctx, const UnparsedArgs& unparsed_args); +int server_main (EmlAppContext& ctx, const UnparsedArgs& unparsed_args); +int space_main (EmlAppContext& ctx, const UnparsedArgs& unparsed_args); +int probe_main (EmlAppContext& ctx, const UnparsedArgs& unparsed_args); + +/****************************************************************************/ + +#if (BOOST_VERSION >= 105900) && (BOOST_VERSION <= 106400) +template +struct greedy_typed_value : public boost::program_options::typed_value { + explicit greedy_typed_value() : boost::program_options::typed_value(nullptr) {} + bool adjacent_tokens_only() const override { return false; } + unsigned max_tokens() const override { return 1; } +}; +template boost::program_options::typed_value* greedy_value() { + return new greedy_typed_value(); +} +#else +template boost::program_options::typed_value* greedy_value() { + return boost::program_options::value(); +} +#endif + +/****************************************************************************/ + +inline boost::program_options::typed_value* txtArg() { + return boost::program_options::value(); +} + +inline boost::program_options::typed_value* txtArg(const std::string& default_value) { + return txtArg()->default_value(default_value); +} + +inline boost::program_options::typed_value* txtArgOpt(const std::string& implicit_value = "") { + return greedy_value()->implicit_value(implicit_value); +} + +inline boost::program_options::typed_value* intArg() { + return boost::program_options::value(); +} + +inline boost::program_options::typed_value* intArg(int default_value) { + return boost::program_options::value()->default_value(default_value); +} + +/****************************************************************************/ + +std::string outputFilename( + const EmlAppContext& ctx, const std::string& input, + const char* dash_suffix = nullptr, const char* extension = nullptr +); + +size_t parseByteSize(const std::string& spec); +std::pair parseByteSizes(const std::string& spec); + +/****************************************************************************/ + +} // oes::apps::eml diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EtpClient12.cpp b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EtpClient12.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d3c963fc903c165aa06e285b1a17558a596928ff --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EtpClient12.cpp @@ -0,0 +1,371 @@ +// ============================================================================ +// Copyright 2016-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "EtpClient12.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace oes::apps::eml { + +using oes::eml::etp12::rpcFor; +using oes::eml::etp12::Datatypes::ErrorInfo; + +//============================================================================ + +EtpClient12::EtpClient12( + const EmlAppContext& ctx +) : ctx_(ctx) +{ +} + +bool EtpClient12::config() { + const std::string& timeouts_spec = ctx_.stringArg("timeout"); + auto timeouts = oes::core::parseCsvSimpleDurationPair(timeouts_spec); + const std::chrono::milliseconds zero_ms(0); + if (timeouts.first < zero_ms || timeouts.second < zero_ms) { + ctx_.err() << "Error: Invalid --timeout. Unknown time unit (accepts s, ms, and min only)." << std::endl; + return false; + } + if (timeouts.first == zero_ms || timeouts.second == zero_ms) { + ctx_.err() << "Error: Invalid --timeout. Cannot be empty or zero." << std::endl; + return false; + } + connect_timeout_ = timeouts.first; + put_get_timeout_ = timeouts.second; + + cfg_.verbose_ = ctx_.verbose; + cfg_.debug_ = ctx_.debug; + cfg_.quiet_ = ctx_.quiet; + + if (!ctx_.hasArg("server-url")) { + ctx_.err() << "Error: Missing --server-url option" << std::endl; + return false; + } + + if (ctx_.hasArg("etp-trace")) { + cfg_.enable_tracing_ = true; + cfg_.trace_file_ = ctx_.stringArg("etp-trace"); + } + + std::string space_path = ctx_.stringArg("space-path"); + + const std::string& mms_spec = ctx_.stringArg("max-msg-size"); + if (!mms_spec.empty()) try { + cfg_.max_mgs_size_ = parseByteSize(mms_spec); + } catch (const std::runtime_error& ex) { + ctx_.err() << "Error: Invalid --max-msg-size option: " << ex.what() << std::endl; + return false; + } + + const int mqd = ctx_.args["msg-queue-depth"].as(); + cfg_.msg_queue_depth_ = (mqd >= 0)? mqd: -mqd; + cfg_.advertise_client_queue_depth_ = (mqd >= 0); + + if (!(0 < cfg_.msg_queue_depth_ && cfg_.msg_queue_depth_ <= 100)) { + ctx_.err() << "Error: Invalid --msg-queue-depth option: Not in range [1, 100]" << std::endl; + return false; + } + + auto warnWhenSet = [this](std::vector args) { + for (const auto& arg : args) { + if (ctx_.hasArg(arg)) { + ctx_.out() << "Warning: Option --" << arg << " ignored" << std::endl; + } + } + }; + auto warnWhenEmpty = [this](std::vector args) { + for (const auto& arg : args) { + if (ctx_.stringArg(arg).empty()) { + ctx_.out() << "Warning: No --" << arg << " option" << std::endl; + } + } + }; + + std::string auth = ctx_.stringArg("auth"); + + if (auth == "auto") { + if (ctx_.hasArg("jwt-token") || ctx_.hasArg("jwt-secret")) { + auth = "bearer"; + } else if (ctx_.hasArg("username") || ctx_.hasArg("password")) { + auth = "basic"; + } else { + auth = "none"; + } + } + + if (auth == "none") { + warnWhenSet({ "username", "password", "jwt-secret", "jwt-token" }); + if (ctx_.stringArg("auth") != "none") { // i.e. not --auth=auto + ctx_.msg() << "Warning: No authentication, since no --username or --password" << std::endl; + } + cfg_.auth_hdr_.clear(); + } else if (auth == "basic") { + warnWhenSet({ "jwt-secret", "jwt-token" }); + warnWhenEmpty({ "username", "password" }); + + std::string username = ctx_.stringArg("username"); + std::string password = ctx_.stringArg("password"); + cfg_.withBasicAuth(username, password); + } else if (auth == "bearer") { + if (ctx_.hasArg("jwt-token")) { + warnWhenSet({ "username", "password", "jwt-secret" }); + + cfg_.withJwtAuth(ctx_.stringArg("jwt-token")); + } else if (ctx_.hasArg("jwt-secret")) { + warnWhenSet({ "jwt-token" }); + warnWhenEmpty({ "username", "password" }); + + std::string username = ctx_.stringArg("username"); + std::string password = ctx_.stringArg("password"); + cfg_.withJwtAuth(username, password, ctx_.stringArg("jwt-secret")); + } else { + ctx_.err() << "Error: Bearer authentication requires --jwt-secret or --jwt-token" << std::endl; + return false; + } + } else { + assert(auth != "auto"); + ctx_.err() << "Error: Invalid --auth option: " + "Use 'auto', 'none', 'basic, or 'bearer'" << std::endl; + return false; + } + + if (ctx_.hasArg("no-http-headers")) { + cfg_.emulate_browser_client_ = true; + } + + if (ctx_.hasArg("acknowledge")) { + cfg_.acknowledge_ = true; + } + + cfg_.compress_ = ctx_.hasArg("compress"); + if (cfg_.compress_) { + std::string zcomp_csv = ctx_.stringArg("compress"); + std::vector zcomp_args; + boost::split(zcomp_args, zcomp_csv, boost::algorithm::is_any_of(",;|")); + // Check compressor names perhaps + cfg_.zcompressors_ = zcomp_args; + } + + if (ctx_.hasArg("shun-protocols")) { + // Split the arg's value + std::string shun_csv = ctx_.stringArg("shun-protocols"); + std::vector shun_args; + boost::split(shun_args, shun_csv, boost::algorithm::is_any_of(",;|")); + + // Build the set of valid values to be shunned. + // Accept Store (the name), Store(4) (the label), 4 (just the ID). + const auto& proto_map = oes::eml::etp12::protocols(); + using ProtoEnum = oes::eml::etp12::Datatypes::Protocol; + std::map accepted; + for (const auto& entry : proto_map) { + const oes::eml::etp12::ProtocolInfo& pinfo = entry.second; + const auto proto_id = ProtoEnum(entry.first); + if (proto_id != ProtoEnum::eCore) { + accepted.emplace(boost::to_lower_copy(pinfo.name_), proto_id); + accepted.emplace(boost::to_lower_copy(pinfo.label()), proto_id); + accepted.emplace(std::to_string(int(proto_id)), proto_id); + } + } + // Add my own "shorter" aliases too + accepted.emplace("types", ProtoEnum::eSupportedTypes); + accepted.emplace("disco", ProtoEnum::eDiscovery); + accepted.emplace("trans", ProtoEnum::eTransaction); + accepted.emplace("space", ProtoEnum::eDataspace); + accepted.emplace("array", ProtoEnum::eDataArray); + + std::set shunned; + for (const std::string& arg : shun_args) { + auto shun_arg = boost::to_lower_copy(arg); + boost::trim(shun_arg); + auto found = accepted.find(shun_arg); + if (found != accepted.end()) { + ProtoEnum proto_id = found->second; + if (proto_id == ProtoEnum::eCore) { + ctx_.err() << "Error: Cannot shun Core(0) protocol" << std::endl; + return false; + } + if (shunned.emplace(proto_id).second) { + auto proto_found = proto_map.find(int(proto_id)); + assert(proto_found != proto_map.end()); + ctx_.dbg() << "Shun-Protocol " << proto_found->second.label() << std::endl; + } + } else { + ctx_.err() << "Error: Cannot shun unknown protocol: " << arg << std::endl; + return false; + } + } + + if (shunned.empty()) { + ctx_.err() << "Error: Invalid --shun-protocols option: empty" << std::endl; + return false; + } + + std::vector requested; + for (auto proto : cfg_.requested_protocols_) { + if (shunned.find(proto) == shunned.end()) { + requested.emplace_back(proto); + } else { + auto proto_found = proto_map.find(int(proto)); + assert(proto_found != proto_map.end()); + ctx_.out() << "Warning: Shunning " << proto_found->second.label() + << " protocol" << std::endl; + } + } + cfg_.requested_protocols_.swap(requested); + + if (cfg_.requested_protocols_.empty()) { + ctx_.out() << "Warning: All requested protocols were shunned" << std::endl; + return false; + } + } + + cfg_.server_url_ = ctx_.stringArg("server-url"); + + return true; +} + +bool EtpClient12::connect() { + if (!config()) { + return false; + } + + clt_.reset(new etp::Client(cfg_)); + ErrorInfo err; + if (!clt_->connect(connect_timeout_, &err)) { + ctx_.err() << "Error: Connect: " << cfg_.server_url_ + << " #" << err.m_code << ": " << err.m_message << std::endl; + return false; + } + return true; +} + +std::vector EtpClient12::globSpaces( + const std::string& pattern, size_t* total +) { + return globSpacesSince(pattern, 0, total); +} + +std::vector EtpClient12::globSpacesSince( + const std::string& pattern, int64_t lastChangedFilter, size_t* total +) { + auto* proto = space_proto(); + if (!proto) { + throw ActionFailed(); + } + + auto rpc = rpcFor(proto, &etp::proto12::IDataspace::GetDataspaces); + + if (lastChangedFilter) { + rpc.req_.m_storeLastWriteFilter = lastChangedFilter; + } + + auto ok = rpc.call(ctx_.err(), put_get_timeout_); + if (!ok.first) { + throw ActionFailed(); + } + + auto& spaces = rpc.res_.m_dataspaces; + if (total) { + *total = spaces.size(); + } + + if (!pattern.empty()) { + // TODO No patern matching for now + // Exact match + spaces.erase( + std::remove_if(spaces.begin(), spaces.end(), [pattern](const Dataspace& ds) { + return pattern != ds.m_path; + }), + spaces.end() + ); + } + + return spaces; +} + +//============================================================================ + +options_description connect_options(unsigned terminalWidth, EtpDefaults defaults) { + const int queue_sign = defaults.advertise_client_queue_depth_? +1: -1; + + const std::string& dT = defaults.timeout_; + const std::string& dZ = defaults.zcompressors_; + const std::string& dM = defaults.max_msg_size_; + const int dB = defaults.max_batch_size_; + const int dQ = defaults.msg_queue_depth_ * queue_sign; + + auto zarg = defaults.compress_? txtArg(dZ): txtArgOpt(dZ); + auto zcmt = "Set Compress-Body flag on all ETP requests\n" + "Specify compressor(s) (Default: " + dZ + ")"; + + options_description opts("Connect Options", terminalWidth); + opts.add_options() + ("server-url,S", txtArg(), "ETP 1.2 server URL") + ("etp-trace", txtArgOpt(), "Trace ETP messages to file\n" + "Filename inferred if empty or directory") + + ("timeout", txtArg(dT), "ETP timeouts for CONNECT and GET/PUT requests") + ("max-msg-size,M", txtArg(dM), "Maximum WebSocket message size(s) in bytes") + ("max-batch-size,B", intArg(dB), "Maximum parts per multi-part message") + ("msg-queue-depth,Q", intArg(dQ), "Maximum concurrent (in-flight) messages") + ("acknowledge", "Set Acknowledge flag on all ETP requests") + + // On client-side, compression is opt-in, while + // on server-side, compression is always active, BUT depends on client + ("compress,Z", zarg, zcmt.c_str()) + + // Allows client-side to simulate a server not implementing a protocol + // the server does in fact support, by not requesting it. For testing only. + // E.g. `space --stats` normally uses SupportedTypes. Shunning SupportedTypes + // allows testing its (slower) fallback using Discovery protocol instead. + ("shun-protocols", txtArg(), "Force client to ignore those protocols") + // ("shun-extensions", txtArg(), "Force client to ignore those extensions") // TODO + // ("hide-capabilities", txtArg(), "Force endpoint to hide its capabilities") // TODO + ; + return opts; +} + +//============================================================================ + +options_description authenticate_options(unsigned terminalWidth) { + options_description opts("Authentication Options", terminalWidth); + opts.add_options() + // Auth-related options + ("auth,A", txtArg("auto"), "Authentication method: 'auto, 'none', 'basic', 'bearer'.") + ("username,u", txtArg(), "User's name to authenticate with.") + ("password,p", txtArg(), "User's password to authenticate with.") + ("jwt-token,T", txtArg(), "Explicit JWT token. Used as-is.") + ("jwt-secret,k", txtArg(), "JWT Signing Key. Uses Basic Authentication if omitted.") + ("no-http-headers", "Use URL for AuthN. Emulates Browser-based clients.") + ; + return opts; +} + +//============================================================================ + +} // namespace oes::apps::eml diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EtpClient12.h b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EtpClient12.h new file mode 100644 index 0000000000000000000000000000000000000000..44b059ea7841b67c2732c839a189f29de14bcac0 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/EtpClient12.h @@ -0,0 +1,96 @@ +// ============================================================================ +// Copyright 2016-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#pragma once + +#include "EmlAppContext.h" + +#include +#include + +#include +#include + +namespace oes::apps::eml { + +namespace etp = oes::eml::etp12; +using etp::Datatypes::Object::Dataspace; + +//============================================================================ + +struct EtpDefaults { + std::string timeout_ = "10s,10s"; + std::string max_msg_size_ = "1MB"; + + int max_batch_size_ = 100; + int msg_queue_depth_ = 5; + + bool compress_ = false; + std::string zcompressors_ = "gzip"; // TODO: lz4, zstd + + // i.e. whether to enabling blocking server-side on too many concurrent messages + // (when false), or instead error-out (when true), the latter implying the client + // paces itself to not go over the server-advertised max_queue_size. + bool advertise_client_queue_depth_ = true; +}; + +options_description connect_options(unsigned terminalWidth, EtpDefaults defaults = {}); +options_description authenticate_options(unsigned terminalWidth); + +//============================================================================ + +struct EtpClient12 { + const EmlAppContext& ctx_; + etp::ClientConfig cfg_; + std::unique_ptr clt_; + + std::chrono::milliseconds connect_timeout_; + std::chrono::milliseconds put_get_timeout_; + + EtpClient12(const EmlAppContext& ctx); + + bool config(); + bool connect(); + + std::vector globSpaces( + const std::string& pattern, size_t* total = nullptr + ); + std::vector globSpacesSince( + const std::string& pattern, int64_t lastChangedFilter = 0, size_t* total = nullptr + ); + + template + TProto* get_proto() { + assert(clt_); + if (auto* proto = clt_->get_proto()) { + return proto; + } + ctx_.err() << "Error: " << cfg_.server_url_ << ": No " + << TProto::protocolName() << " protocol support" << std::endl; + return nullptr; + } + + auto core_proto() { return get_proto(); } + auto store_proto() { return get_proto(); } + auto array_proto() { return get_proto(); } + auto disco_proto() { return get_proto(); } + auto types_proto() { return get_proto(); } + auto space_proto() { return get_proto(); } + auto trans_proto() { return get_proto(); } +}; + +//============================================================================ + +} // namespace oes::apps::eml diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/Hdf5Cmds.cpp b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/Hdf5Cmds.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8c536b07b37135fc02a2bbab2a75745316d6a517 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/Hdf5Cmds.cpp @@ -0,0 +1,1137 @@ +// ============================================================================ +// Copyright 2020-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "EmlAppContext.h" + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using std::endl; +using namespace oes::eml; +using namespace oes::apps::eml; +namespace po = boost::program_options; +namespace fs = std::filesystem; + +namespace { + +using SimpleTimer = oes::core::HighResTimer; +using oes::core::thou; + +enum class Hdf5ObjType : unsigned char { + eUNKNOWN = 0, + eGROUP, + eDATASET, + eNAMED_DATATYPE, +}; +std::array hdf5ObTypes() { + return { + Hdf5ObjType::eUNKNOWN, + Hdf5ObjType::eGROUP, + Hdf5ObjType::eDATASET, + Hdf5ObjType::eNAMED_DATATYPE, + }; +} +const char* hdf5ObTypeToString(Hdf5ObjType type) { + switch (type) { + case Hdf5ObjType::eUNKNOWN: return "Unknown Object"; + case Hdf5ObjType::eGROUP: return "Group"; + case Hdf5ObjType::eDATASET: return "Dataset"; + case Hdf5ObjType::eNAMED_DATATYPE: return "Named Datatype"; + } + assert(false); + return "Unknown Object"; +} + +enum class Hdf5DataType : unsigned char { + eUNKNOWN = 0, + eINTEGER, + eFLOAT, + eTIME, + eSTRING, + eBITFIELD, + eOPAQUE, + eCOMPOUND, + eREFERENCE, + eENUM, + eVLEN, + eARRAY, +}; +std::array hdf5DataTypes() { + return { + Hdf5DataType::eUNKNOWN, + Hdf5DataType::eINTEGER, + Hdf5DataType::eFLOAT, + Hdf5DataType::eTIME, + Hdf5DataType::eSTRING, + Hdf5DataType::eBITFIELD, + Hdf5DataType::eOPAQUE, + Hdf5DataType::eCOMPOUND, + Hdf5DataType::eREFERENCE, + Hdf5DataType::eENUM, + Hdf5DataType::eVLEN, + Hdf5DataType::eARRAY, + }; +} +const char* hdf5DataTypeToString(Hdf5DataType type) { + switch (type) { + case Hdf5DataType::eUNKNOWN: return "Unknown"; + case Hdf5DataType::eINTEGER: return "Integer"; + case Hdf5DataType::eFLOAT: return "Floating-Point"; + case Hdf5DataType::eTIME: return "Time"; + case Hdf5DataType::eSTRING: return "String"; + case Hdf5DataType::eBITFIELD: return "Bitfield"; + case Hdf5DataType::eOPAQUE: return "Opaque"; + case Hdf5DataType::eCOMPOUND: return "Compound"; + case Hdf5DataType::eREFERENCE: return "Reference"; + case Hdf5DataType::eENUM: return "Enum"; + case Hdf5DataType::eVLEN: return "Variable-Length"; + case Hdf5DataType::eARRAY: return "Array"; + } + assert(false); + return "Unknown"; +} + +enum class Hdf5NumericType : unsigned char { + eNIL = 0, + eI08, + eU08, + eI16, + eU16, + eI32, + eU32, + eI64, + eU64, + eF32, + eF64, +}; +std::array hdf5NumTypes() { + return { + Hdf5NumericType::eI08, + Hdf5NumericType::eU08, + Hdf5NumericType::eI16, + Hdf5NumericType::eU16, + Hdf5NumericType::eI32, + Hdf5NumericType::eU32, + Hdf5NumericType::eI64, + Hdf5NumericType::eU64, + Hdf5NumericType::eF32, + Hdf5NumericType::eF64, + }; +} +const char* hdf5NumTypeToString(Hdf5NumericType type) { + switch (type) { + case Hdf5NumericType::eNIL: break; + case Hdf5NumericType::eI08: return "char/bool"; + case Hdf5NumericType::eU08: return "byte"; + case Hdf5NumericType::eI16: return "short"; + case Hdf5NumericType::eU16: return "ushort"; + case Hdf5NumericType::eI32: return "int"; + case Hdf5NumericType::eU32: return "uint"; + case Hdf5NumericType::eI64: return "long"; + case Hdf5NumericType::eU64: return "ulong"; + case Hdf5NumericType::eF32: return "float"; + case Hdf5NumericType::eF64: return "double"; + } + assert(false); + return "Unknown"; +} + +// TODO +#if 0 +typedef enum H5D_layout_t { + H5D_LAYOUT_ERROR = -1, + + H5D_COMPACT = 0, /*raw data is very small */ + H5D_CONTIGUOUS = 1, /*the default */ + H5D_CHUNKED = 2, /*slow and fancy */ + H5D_VIRTUAL = 3, /*actual data is stored in other datasets */ + H5D_NLAYOUTS = 4 /*this one must be last! */ +} H5D_layout_t; + +/* Types of chunk index data structures */ +typedef enum H5D_chunk_index_t { + H5D_CHUNK_IDX_BTREE = 0, /* v1 B-tree index (default) */ + H5D_CHUNK_IDX_SINGLE = 1, /* Single Chunk index (cur dims[]=max dims[]=chunk dims[]; filtered & non-filtered) */ + H5D_CHUNK_IDX_NONE = 2, /* Implicit: No Index (H5D_ALLOC_TIME_EARLY, non-filtered, fixed dims) */ + H5D_CHUNK_IDX_FARRAY = 3, /* Fixed array (for 0 unlimited dims) */ + H5D_CHUNK_IDX_EARRAY = 4, /* Extensible array (for 1 unlimited dim) */ + H5D_CHUNK_IDX_BT2 = 5, /* v2 B-tree index (for >1 unlimited dims) */ + H5D_CHUNK_IDX_NTYPES /* This one must be last! */ +} H5D_chunk_index_t; + +/* Values for the space allocation time property */ +typedef enum H5D_alloc_time_t { + H5D_ALLOC_TIME_ERROR = -1, + H5D_ALLOC_TIME_DEFAULT = 0, + H5D_ALLOC_TIME_EARLY = 1, + H5D_ALLOC_TIME_LATE = 2, + H5D_ALLOC_TIME_INCR = 3 +} H5D_alloc_time_t; +#endif//0 + +struct ScopedH5Dataset { + hid_t dataset_ = -1; + hid_t filespace_ = -1; + hid_t data_type_ = -1; + + ~ScopedH5Dataset() { + if (data_type_ >= 0) { + H5Tclose(data_type_); + } + if (filespace_ >= 0) { + H5Sclose(filespace_); + } + if (dataset_ >= 0) { + H5Dclose(dataset_); + } + } + + bool read(std::vector& buf) const { + // TODO: Try disable chunk caching, H5Pset_chunk_cache + // to avoid using 1MB per dataset, even after we close it. + herr_t read_rc = H5Dread( + dataset_, data_type_, H5S_ALL, + filespace_, H5P_DEFAULT, buf.data() + ); + return read_rc >= 0; + }; + + bool read( + std::vector& buf, + const std::vector& starts, + const std::vector& counts + ) const { + herr_t rc = H5Sselect_hyperslab( + filespace_, H5S_SELECT_SET, + starts.data(), nullptr, counts.data(), nullptr + ); + if (rc >= 0) { + H5::DataSpace mem_space; + hsize_t slab_size = etp12::productOf(counts); + mem_space.setExtentSimple(1, &slab_size, &slab_size); + assert(slab_size <= buf.capacity()); + buf.resize(slab_size); + + rc = H5Dread( + dataset_, data_type_, mem_space.getId(), + filespace_, H5P_DEFAULT, buf.data() + ); + } + return rc >= 0; + }; +}; + +constexpr int max_rank = 4; + +struct Hdf5ObjInfo { + std::string name_; + + Hdf5ObjType type_ = Hdf5ObjType::eUNKNOWN; + Hdf5DataType data_type_ = Hdf5DataType::eUNKNOWN; + unsigned char type_bytes_ = 0; + bool type_sign_ = true; + + unsigned char rank_ = 0; + std::array dims_{}; + + unsigned header_size_ = 0; + unsigned attr_count_ = 0; + + size_t size_ = 0; + size_t attr_size_ = 0; + size_t storage_size_ = 0; + + + bool is_numeric() const { + switch (data_type_) { + case Hdf5DataType::eINTEGER: + return 1 <= type_bytes_ && type_bytes_ <= 8; + case Hdf5DataType::eFLOAT: + return type_bytes_ == 4 || type_bytes_ == 8; + default: + break; + } + return false; + } + + Hdf5NumericType numeric_type() const { + switch (data_type_) { + case Hdf5DataType::eINTEGER: + if (type_sign_) { + switch (type_bytes_) { + case 1: return Hdf5NumericType::eI08; + case 2: return Hdf5NumericType::eI16; + case 4: return Hdf5NumericType::eI32; + case 8: return Hdf5NumericType::eI64; + default: return Hdf5NumericType::eNIL; + } + } else { + switch (type_bytes_) { + case 1: return Hdf5NumericType::eU08; + case 2: return Hdf5NumericType::eU16; + case 4: return Hdf5NumericType::eU32; + case 8: return Hdf5NumericType::eU64; + default: return Hdf5NumericType::eNIL; + } + } + break; // unreachable + case Hdf5DataType::eFLOAT: + if (type_bytes_ == 4) { + return Hdf5NumericType::eF32; + } else if (type_bytes_ == 8) { + return Hdf5NumericType::eF64; + } + break; // unreachable + default: + break; + } + return Hdf5NumericType::eNIL; + }; + + std::string dims() const { + std::ostringstream oss; + for (unsigned char i = 0; i < rank_; ++i) { + oss << thou(dims_[i]) << " x "; + } + auto s = oss.str(); + if (!s.empty()) { + s.resize(s.size() - 3); + } + return s; + } + + size_t uncompressed_bytes() const { + size_t full_size = type_bytes_; + for (unsigned char i = 0; i < rank_; ++i) { + full_size *= dims_[i]; + } + return full_size; + } + + std::string desc(bool ignore_compression_factor = false) const { + const auto usize = uncompressed_bytes(); + return fmt::format( + "{}D - {} - {} ({} bytes, {:.1f}x)", + rank_, dims(), is_numeric()? + hdf5NumTypeToString(numeric_type()): + hdf5DataTypeToString(data_type_), + thou(usize), ignore_compression_factor? + 1.0: (1.0 * usize / storage_size_) + ); + } + + std::unique_ptr open(hid_t file_id) const { + assert(!name_.empty()); + + auto res = std::make_unique(); + + res->dataset_ = H5Dopen2(file_id, name_.c_str(), H5P_DEFAULT); + if (res->dataset_ < 0) { + return nullptr; + } + + res->filespace_ = H5Dget_space(res->dataset_); + if (res->filespace_ < 0) { + return nullptr; + } + + res->data_type_ = H5Dget_type(res->dataset_); + if (res->data_type_ < 0) { + return nullptr; + } + + return res; + } +}; + +struct Hdf5FileInfo { + bool datasets_only_ = false; + bool get_dataspace_ = false; + std::vector objects_; + + size_t size_per_type_[int(Hdf5ObjType::eNAMED_DATATYPE) + 1] = { 0 }; + size_t count_per_type_[int(Hdf5ObjType::eNAMED_DATATYPE) + 1] = { 0 }; + size_t header_size_per_type_[int(Hdf5ObjType::eNAMED_DATATYPE) + 1] = { 0 }; + size_t attr_count_per_type_[int(Hdf5ObjType::eNAMED_DATATYPE) + 1] = { 0 }; + size_t count_per_data_type_[int(Hdf5DataType::eARRAY) + 1] = { 0 }; + size_t count_per_rank_and_num_type_[max_rank + 1][int(Hdf5NumericType::eF64) + 1] = { 0 }; + size_t count_per_rank_[max_rank + 1] = { 0 }; + size_t dataset_storage_size_ = 0; + size_t dataset_uncompressed_size_ = 0; + + size_t sizeOf(Hdf5ObjType type) const { + return size_per_type_[int(type)]; + } + size_t countOf(Hdf5ObjType type) const { + return count_per_type_[int(type)]; + } + size_t attrCountOf(Hdf5ObjType type) const { + return attr_count_per_type_[int(type)]; + } + size_t headerSizeOf(Hdf5ObjType type) const { + return header_size_per_type_[int(type)]; + } +}; + +static herr_t acceptHdf5ObjInfo( + hid_t loc_id, const char* name, const H5O_info_t* linfo, void* opdata +) { + const bool is_dataset = (linfo->type == H5O_TYPE_DATASET); + + Hdf5FileInfo* file_info = reinterpret_cast(opdata); + if (file_info->datasets_only_ && !is_dataset) { + return 0; // i.e. continue iterating / visiting + } + + Hdf5ObjInfo& oinfo = file_info->objects_.emplace_back(); + + if (name[0] == '.' && name[1] == '\0') { + oinfo.name_.clear(); // the root. We'll prepend slash later + } else { + oinfo.name_.assign(name); // NOT prefixed with slash + } + oinfo.header_size_ = static_cast(linfo->hdr.space.total); + oinfo.attr_count_ = static_cast(linfo->num_attrs); + + oinfo.size_ = linfo->meta_size.obj.heap_size + linfo->meta_size.obj.index_size; + oinfo.attr_size_ = linfo->meta_size.attr.heap_size + linfo->meta_size.attr.index_size; + + switch (linfo->type) { + case H5O_TYPE_UNKNOWN: oinfo.type_ = Hdf5ObjType::eUNKNOWN; break; + case H5O_TYPE_GROUP: oinfo.type_ = Hdf5ObjType::eGROUP; break; + case H5O_TYPE_DATASET: oinfo.type_ = Hdf5ObjType::eDATASET; break; + case H5O_TYPE_NAMED_DATATYPE: oinfo.type_ = Hdf5ObjType::eNAMED_DATATYPE; break; + default: oinfo.type_ = Hdf5ObjType::eUNKNOWN; break; + } + + const int idx = int(oinfo.type_); + file_info->size_per_type_ [idx] += (oinfo.size_ + oinfo.attr_size_); + file_info->count_per_type_ [idx] += 1; + file_info->header_size_per_type_ [idx] += oinfo.header_size_; + file_info->attr_count_per_type_ [idx] += oinfo.attr_count_; + + if (file_info->get_dataspace_ && is_dataset) { + hid_t dataset = H5Dopen1(loc_id, name); + hid_t filespace = H5Dget_space(dataset); + oinfo.rank_ = static_cast(H5Sget_simple_extent_ndims(filespace)); + if (oinfo.rank_ <= max_rank) { + hsize_t dims[max_rank]{}; + hsize_t maxdims[max_rank]{}; + H5Sget_simple_extent_dims(filespace, dims, maxdims); + for (auto i = 0; i < oinfo.rank_; ++i) { + oinfo.dims_[i] = static_cast(dims[i]); + } + } + oinfo.storage_size_ = H5Dget_storage_size(dataset); + + hid_t data_type = H5Dget_type(dataset); + const size_t tsize = H5Tget_size(data_type); + oinfo.type_bytes_ = (tsize > 255)? 255: (unsigned char)(tsize); + oinfo.type_sign_ = H5Tget_sign(data_type)? true: false; + + switch (H5Tget_class(data_type)) { + case H5T_NO_CLASS: oinfo.data_type_ = Hdf5DataType::eUNKNOWN; break; + case H5T_INTEGER: oinfo.data_type_ = Hdf5DataType::eINTEGER; break; + case H5T_FLOAT: oinfo.data_type_ = Hdf5DataType::eFLOAT; break; + case H5T_TIME: oinfo.data_type_ = Hdf5DataType::eTIME; break; + case H5T_STRING: oinfo.data_type_ = Hdf5DataType::eSTRING; break; + case H5T_BITFIELD: oinfo.data_type_ = Hdf5DataType::eBITFIELD; break; + case H5T_OPAQUE: oinfo.data_type_ = Hdf5DataType::eOPAQUE; break; + case H5T_COMPOUND: oinfo.data_type_ = Hdf5DataType::eCOMPOUND; break; + case H5T_REFERENCE: oinfo.data_type_ = Hdf5DataType::eREFERENCE; break; + case H5T_ENUM: oinfo.data_type_ = Hdf5DataType::eENUM; break; + case H5T_VLEN: oinfo.data_type_ = Hdf5DataType::eVLEN; break; + case H5T_ARRAY: oinfo.data_type_ = Hdf5DataType::eARRAY; break; + case H5T_NCLASSES: oinfo.data_type_ = Hdf5DataType::eUNKNOWN; break; + } + + H5Dclose(dataset); + + file_info->count_per_rank_[oinfo.rank_] += 1; + file_info->count_per_data_type_[int(oinfo.data_type_)] += 1; + file_info->dataset_storage_size_ += oinfo.storage_size_; + file_info->dataset_uncompressed_size_ += oinfo.uncompressed_bytes(); + if (oinfo.is_numeric()) { + file_info->count_per_rank_and_num_type_[oinfo.rank_][int(oinfo.numeric_type())] += 1; + } + } + + return 0; // i.e. continue iterating / visiting +} + +bool hdf5_info(const EmlAppContext& ctx) { + std::string h5_filename = ctx.stringArg(ctx.action()); + + const H5::FileAccPropList& fapl = H5::FileAccPropList(); + H5::H5File file(h5_filename, H5F_ACC_RDONLY, H5::FileCreatPropList::DEFAULT, fapl); + + Hdf5FileInfo file_info; + const herr_t rc = H5Ovisit( + file.getId(), H5_INDEX_NAME, H5_ITER_NATIVE, &acceptHdf5ObjInfo, &file_info + ); + if (rc < 0) { + ctx.err() << "Error: Failed to scan HDF5 file: " << h5_filename << endl; + return false; + } + + auto& os = ctx.out(); + for (auto type : hdf5ObTypes()) { + if (size_t n = file_info.countOf(type)) { + os << fmt::format( + "{:>18} {}{}", thou(n), hdf5ObTypeToString(type), ((n> 1)? "s": "") + ) << endl; + } + } + for (auto type : hdf5ObTypes()) { + if (size_t n = file_info.attrCountOf(type)) { + os << fmt::format( + "{:>18} {} attribute{}", thou(n), hdf5ObTypeToString(type), ((n> 1)? "s": "") + ) << endl; + } + } + for (auto type : hdf5ObTypes()) { + if (size_t n = file_info.headerSizeOf(type)) { + os << fmt::format( + "{:>18} {} header byte{}", thou(n), hdf5ObTypeToString(type), ((n> 1)? "s": "") + ) << endl; + } + } + for (auto type : hdf5ObTypes()) { + if (size_t n = file_info.sizeOf(type)) { + os << fmt::format( + "{:>18} {} (index & heap) byte{}", thou(n), hdf5ObTypeToString(type), ((n> 1)? "s": "") + ) << endl; + } + } + + // counts of values of given byte-size-range, by increasing powers of two. + // from 0, 1, 2, 4, 8, 16, 32, 64, ... , 1KB, ... 1MB, ..., 32MB, etc... + std::vector bytes_histogram_; + std::vector datasets; + bytes_histogram_.reserve(32); + datasets.reserve(file_info.countOf(Hdf5ObjType::eDATASET)); + + std::map group2count; + std::map group_name2count; + std::map dataset_name2count; + for (const Hdf5ObjInfo& oinfo : file_info.objects_) { + size_t last_slash = oinfo.name_.rfind('/'); + if (last_slash != std::string::npos) { + ++group2count[oinfo.name_.substr(0, last_slash)]; + + if (oinfo.type_ == Hdf5ObjType::eDATASET) { + ++dataset_name2count[oinfo.name_.substr(last_slash + 1)]; + } else { + ++group_name2count[oinfo.name_.substr(last_slash + 1)]; + } + } else { + ++group2count[std::string()]; + if (oinfo.type_ == Hdf5ObjType::eDATASET) { + ++dataset_name2count[std::string()]; + } else { + ++group_name2count[std::string()]; + } + } + + if (oinfo.type_ == Hdf5ObjType::eDATASET) { + datasets.emplace_back(&oinfo); + + // Not sure 0-size is normal... + const size_t bucket_idx = (oinfo.size_ == 0)? 0: + 1 + static_cast(std::log2(oinfo.size_)); + + if (bucket_idx >= bytes_histogram_.size()) { + bytes_histogram_.resize(bucket_idx + 1); + } + ++bytes_histogram_[bucket_idx]; + } + } + + const size_t topN = ctx.args["top"].as(); // 10 by default + + os << fmt::format("\n{:>18} by 'overhead' size-range >>>", "<<< Datasets") << endl; + if (!bytes_histogram_.empty() && bytes_histogram_[0]) { + os << fmt::format("{:>18} empty", thou(bytes_histogram_[0])) << endl; + } + for (size_t i = 1; i < bytes_histogram_.size(); ++i) { + const size_t width = thou(size_t(pow(2, bytes_histogram_.size() - 1))).size(); + const size_t count = bytes_histogram_[i]; + std::ostream& os2 = count? os: ctx.msg(); + os2 << fmt::format( // show empty range in verbose mode only + "{:>18} ({:>{}} - {:>{}}] bytes", thou(count), + thou(size_t(pow(2, i - 1))), width, + thou(size_t(pow(2, i))), width + ) << endl; + } + + using map_entry = std::map::value_type; + std::vector dataset_names; + dataset_names.reserve(dataset_name2count.size()); + for (const map_entry& entry : dataset_name2count) { + dataset_names.emplace_back(&entry); + } + const size_t topN_byName = std::min<>(topN, dataset_names.size()); + std::partial_sort( + dataset_names.begin(), dataset_names.begin() + topN_byName, dataset_names.end(), + [](const map_entry* lhs, const map_entry* rhs) { + return lhs->second > rhs->second; + } + ); + std::stable_sort( // show them alphabetically though, nicer + dataset_names.begin(), dataset_names.begin() + topN_byName, + [](const map_entry* lhs, const map_entry* rhs) { + return lhs->first < rhs->first; + } + ); + os << fmt::format("\n{:>18} by name - Top {} >>>", "<<< Datasets", topN) << endl; + for (size_t i = 0; i < topN_byName; ++i) { + const map_entry& entry = *dataset_names[i]; + os << fmt::format("{:>18} {}", thou(entry.second), entry.first) << endl; + } + + const size_t topN_bySize = std::min<>(topN, datasets.size()); + std::partial_sort( + datasets.begin(), datasets.begin() + topN_bySize, datasets.end(), + [](const Hdf5ObjInfo* lhs, const Hdf5ObjInfo* rhs) { + return lhs->size_ > rhs->size_; + } + ); + os << fmt::format("\n{:>18} 'overhead' size - Top {} >>>", "<<< Dataset", topN) << endl; + for (size_t i = 0; i < topN_bySize; ++i) { + const Hdf5ObjInfo& oinfo = *datasets[i]; + os << fmt::format("{:>18} {}", thou(oinfo.size_), oinfo.name_) << endl; + } + + return true; +} + +bool hdf5_stats(const EmlAppContext& ctx) { + std::string h5_filename = ctx.stringArg(ctx.action()); + + const H5::FileAccPropList& fapl = H5::FileAccPropList(); + H5::H5File file(h5_filename, H5F_ACC_RDONLY, H5::FileCreatPropList::DEFAULT, fapl); + + Hdf5FileInfo file_info; + file_info.get_dataspace_ = true; + const herr_t rc = H5Ovisit( + file.getId(), H5_INDEX_NAME, H5_ITER_NATIVE, &acceptHdf5ObjInfo, &file_info + ); + if (rc < 0) { + ctx.err() << "Error: Failed to scan HDF5 file: " << h5_filename << endl; + return false; + } + + auto& os = ctx.out(); + os << fmt::format("{:>18} Dataset stored bytes", thou(file_info.dataset_storage_size_)) << endl; + os << fmt::format( + "{:>18} Dataset bytes un-chunked & un-compressed", + thou(file_info.dataset_uncompressed_size_) + ) << endl; + os << fmt::format( + "{:>17.1f}x Compression ratio (exclude metadata overhead)", + 1.0 * file_info.dataset_uncompressed_size_ / file_info.dataset_storage_size_ + ) << endl; + + os << fmt::format("\n{:>18} by rank >>>", "<<< Datasets") << endl; + for (int rank = 0; rank <= max_rank; ++rank) { + if (size_t n = file_info.count_per_rank_[rank]) { + assert(rank); // i.e. nothing at rank 0 + os << fmt::format("{:>18} {}D", thou(n), rank) << endl; + } + } + + os << fmt::format("\n{:>18} by datatype class >>>", "<<< Datasets") << endl; + for (Hdf5DataType dtype : hdf5DataTypes()) { + if (size_t n = file_info.count_per_data_type_[int(dtype)]) { + os << fmt::format("{:>18} {}", thou(n), hdf5DataTypeToString(dtype)) << endl; + } + } + + os << fmt::format("\n{:>18} by rank & numeric type >>>", "<<< Datasets") << endl; + for (int rank = 1; rank <= max_rank; ++rank) + for (Hdf5NumericType ntype : hdf5NumTypes()) { + if (size_t n = file_info.count_per_rank_and_num_type_[rank][int(ntype)]) { + assert(rank); // i.e. nothing at rank 0 + os << fmt::format("{:>18} {}D - {}", thou(n), rank, hdf5NumTypeToString(ntype)) << endl; + } + } + + std::vector bytes_histogram_; + std::vector datasets; + bytes_histogram_.reserve(32); + datasets.reserve(file_info.countOf(Hdf5ObjType::eDATASET)); + for (const Hdf5ObjInfo& oinfo : file_info.objects_) { + if (oinfo.type_ == Hdf5ObjType::eDATASET) { + datasets.emplace_back(&oinfo); + + // Not sure 0-size is normal... + // e.g. 2D point array with 0x3 dimensions. + const size_t bucket_idx = (oinfo.storage_size_ == 0)? 0: + 1 + static_cast(std::log2(oinfo.storage_size_)); + + if (bucket_idx >= bytes_histogram_.size()) { + bytes_histogram_.resize(bucket_idx + 1); + } + ++bytes_histogram_[bucket_idx]; + } + } + + const size_t topN = ctx.args["top"].as(); // 10 by default + + os << fmt::format("\n{:>18} by 'storage' size-range >>>", "<<< Datasets") << endl; + if (!bytes_histogram_.empty() && bytes_histogram_[0]) { + os << fmt::format("{:>18} empty", thou(bytes_histogram_[0])) << endl; + } + for (size_t i = 1; i < bytes_histogram_.size(); ++i) { + const size_t width = thou(size_t(pow(2, bytes_histogram_.size() - 1))).size(); + const size_t count = bytes_histogram_[i]; + std::ostream& os2 = count? os: ctx.msg(); + os2 << fmt::format( // show empty range in verbose mode only + "{:>18} ({:>{}} - {:>{}}] bytes", thou(count), + thou(size_t(pow(2, i - 1))), width, + thou(size_t(pow(2, i))), width + ) << endl; + } + + const size_t topN_bySize = std::min<>(topN, datasets.size()); + std::partial_sort( + datasets.begin(), datasets.begin() + topN_bySize, datasets.end(), + [](const Hdf5ObjInfo* lhs, const Hdf5ObjInfo* rhs) { + return lhs->storage_size_ > rhs->storage_size_; + } + ); + os << fmt::format("\n{:>18} 'storage' size - Top {} >>>", "<<< Dataset", topN) << endl; + for (size_t i = 0; i < topN_bySize; ++i) { + const Hdf5ObjInfo& oinfo = *datasets[i]; + os << fmt::format( + "{:>18} {}\n{:>18} {}", + thou(oinfo.storage_size_), oinfo.name_, "", oinfo.desc() + ) << endl; + } + + return true; +} + +bool hdf5_diff(const EmlAppContext& ctx) { + size_t max_buf_size = 1024*1024; + const std::string& mbs_spec = ctx.stringArg("max-buf-size"); + if (!mbs_spec.empty()) try { + max_buf_size = parseByteSize(mbs_spec); + } catch (const std::runtime_error& ex) { + ctx.err() << "Error: Invalid --max-buf-size option: " << ex.what() << std::endl; + return false; + } + + std::string lhs_filename = ctx.stringArg(ctx.action()); + std::string rhs_filename = ctx.stringArg("with"); + if (rhs_filename.empty()) { + if (!ctx.hasArg("force")) { + ctx.err() << "Error: No --with file to diff against" << std::endl; + return false; + } + rhs_filename = lhs_filename; + } + + const fs::path lhs_path(lhs_filename); + fs::file_status lhs_stat = fs::status(lhs_path); + if (!fs::is_regular_file(lhs_stat)) { + fmt::print(ctx.err(), "Error: File not found : {}\n", lhs_filename); + return false; + } + + if (rhs_filename != lhs_filename) { + const fs::path rhs_path(rhs_filename); + fs::file_status rhs_stat = fs::status(rhs_path); + if (!fs::is_regular_file(rhs_stat)) { + fmt::print(ctx.err(), "Error: File not found : {}\n", rhs_filename); + return false; + } + } else { + fmt::print(ctx.out(), "Warning: Diff'ing {} with itself\n", lhs_filename); + } + + const H5::FileAccPropList fapl; + H5::H5File lhs(lhs_filename, H5F_ACC_RDONLY, H5::FileCreatPropList::DEFAULT, fapl); + H5::H5File rhs(rhs_filename, H5F_ACC_RDONLY, H5::FileCreatPropList::DEFAULT, fapl); + const hid_t lhs_id = lhs.getId(); + const hid_t rhs_id = rhs.getId(); + + Hdf5FileInfo lhs_info; + lhs_info.datasets_only_ = true; + lhs_info.get_dataspace_ = true; + const herr_t lhs_rc = H5Ovisit( + lhs_id, H5_INDEX_NAME, H5_ITER_NATIVE, &acceptHdf5ObjInfo, &lhs_info + ); + if (lhs_rc < 0) { + fmt::print(ctx.err(), "Error: Failed to scan HDF5 file: {}\n", lhs_filename); + return false; + } + const int lhs_width = 2 + static_cast(std::log10(lhs_info.objects_.size())); + fmt::print( + ctx.msg(), "Scanned {:>{}} object in Left file\n", + thou(lhs_info.objects_.size()), lhs_width + ); + + Hdf5FileInfo rhs_info; + rhs_info.datasets_only_ = true; + rhs_info.get_dataspace_ = true; + const herr_t rhs_rc = H5Ovisit( + rhs_id, H5_INDEX_NAME, H5_ITER_NATIVE, &acceptHdf5ObjInfo, &rhs_info + ); + if (rhs_rc < 0) { + fmt::print(ctx.err(), "Error: Failed to scan HDF5 file: {}\n", rhs_filename); + return false; + } + const int rhs_width = 2 + static_cast(std::log10(rhs_info.objects_.size())); + const int width = std::max<>(lhs_width, rhs_width); + fmt::print( + ctx.msg(), "Scanned {:>{}} object in Right file\n", + thou(rhs_info.objects_.size()), width + ); + + auto& lhs_objs = lhs_info.objects_; + auto& rhs_objs = rhs_info.objects_; + + auto lt_by_name = [](const auto& lhs, const auto& rhs) { + return lhs.name_ < rhs.name_; + }; + auto eq_by_name = [](const auto& lhs, const auto& rhs) { + return lhs.name_ == rhs.name_; + }; + + // Sort them lexicographically. + std::sort(lhs_objs.begin(), lhs_objs.end(), lt_by_name); + std::sort(rhs_objs.begin(), rhs_objs.end(), lt_by_name); + + // There should not be any duplicate parts + assert(std::unique(lhs_objs.begin(), lhs_objs.end(), eq_by_name) == lhs_objs.end()); + assert(std::unique(rhs_objs.begin(), rhs_objs.end(), eq_by_name) == rhs_objs.end()); + OES_RELEASE_UNUSED(eq_by_name); + + // Find the in parts not found in out-package. + std::vector missing_dsets; + std::set_difference( + lhs_objs.begin(), lhs_objs.end(), + rhs_objs.begin(), rhs_objs.end(), + std::inserter(missing_dsets, missing_dsets.begin()), + lt_by_name + ); + + // Find the out parts not found in in-package. + std::vector extra_dsets; + std::set_difference( + rhs_objs.begin(), rhs_objs.end(), + lhs_objs.begin(), lhs_objs.end(), + std::inserter(extra_dsets, extra_dsets.begin()), + lt_by_name + ); + + // Find the parts common to both packages. + std::vector common_dsets; + std::set_intersection( + lhs_objs.begin(), lhs_objs.end(), + rhs_objs.begin(), rhs_objs.end(), + std::back_inserter(common_dsets), + lt_by_name + ); + + const size_t match_count = common_dsets.size(); + if (!match_count) { + fmt::print( + ctx.out(), "Nothing in common! ({} vs {} datasets)\n", + thou(lhs_objs.size()), thou(rhs_objs.size()) + ); + return true; + } + + if (!missing_dsets.empty() || !extra_dsets.empty()) { + fmt::print( + ctx.out(), "{} vs {} datasets; {} common; {} missing; {} extra.\n", + thou(lhs_objs.size()), thou(rhs_objs.size()), + thou(match_count), thou(missing_dsets.size()), thou(extra_dsets.size()) + ); + } + + const size_t topN = ctx.args["top"].as(); // 10 by default + + if (!missing_dsets.empty() && topN) { + fmt::print( + ctx.msg(), "\nTop {} (by-name) missing datasets, out of {}:\n", + thou(std::min<>(topN, missing_dsets.size())), thou(missing_dsets.size()) + ); + + size_t count = 0; + for (const Hdf5ObjInfo& obj : missing_dsets) { + fmt::print(ctx.msg(), "- {}: {}\n", obj.name_, obj.desc()); + if (++count >= topN) { + break; + } + } + } + + if (!extra_dsets.empty() && topN) { + fmt::print( + ctx.msg(), "\nTop {} (by-name) extra datasets, out of {}:\n", + thou(std::min<>(topN, extra_dsets.size())), thou(extra_dsets.size()) + ); + + size_t count = 0; + for (const Hdf5ObjInfo& obj : extra_dsets) { + fmt::print(ctx.msg(), "- {}: {}\n", obj.name_, obj.desc()); + if (++count >= topN) { + break; + } + } + } + + size_t diff_count = 0; + auto show_diff_header = [&]() { + if (!diff_count) { + fmt::print( + ctx.msg(), "\nTop {} (by-name) datasets differences:\n", + thou(topN) + ); + } + }; + + size_t checked = 0; + size_t bytes_checked = 0; + size_t io_error_count = 0; + size_t possible_int_conversion_diff = 0; + std::map, size_t> int_convs; + for (const Hdf5ObjInfo& lhs_obj : common_dsets) { + auto rhs_found = std::lower_bound( + rhs_objs.begin(), rhs_objs.end(), lhs_obj, lt_by_name + ); + assert(rhs_found != rhs_objs.end()); + const Hdf5ObjInfo& rhs_obj = *rhs_found; + assert(lhs_obj.name_ == rhs_obj.name_); + ++checked; + + // Use desc as "proxy" for rank, dimension, type, total bytes. + const std::string& lhs_desc = lhs_obj.desc(true); + const std::string& rhs_desc = rhs_obj.desc(true); + if (lhs_desc != rhs_desc) { + show_diff_header(); + fmt::print( + ctx.msg(), "- {}: meta-data diff:\n {}\n {}\n", + lhs_obj.name_, lhs_desc, rhs_desc + ); + + if ((lhs_obj.data_type_ == Hdf5DataType::eINTEGER) && + (rhs_obj.data_type_ == Hdf5DataType::eINTEGER) && + ((lhs_obj.type_bytes_ != rhs_obj.type_bytes_) || + (lhs_obj.type_sign_ != rhs_obj.type_sign_)) + ) { + // Could be an implicit widening of uint to long (lossless), + // or even a ulong to long (possibly lossy). + // TODO: Do a smarter semantic-value comparison, not a byte-comparison + ++possible_int_conversion_diff; + ++int_convs[{ lhs_obj.numeric_type(), rhs_obj.numeric_type() }]; + ++diff_count; + continue; // i.e. Don't stop diff'ing for those particular diffs + } + + if (++diff_count >= topN) { + break; + } + continue; + } + + // Both datasets have the same "shape". Now compare their (uncompressed) bytes + const auto& obj = lhs_obj; + const size_t bytes = obj.uncompressed_bytes(); + + if (!bytes) { + continue; // yes, empty datasets do happen... + } + + assert(lhs_obj.rank_ == rhs_obj.rank_); + assert(lhs_obj.dims_ == rhs_obj.dims_); + assert(lhs_obj.type_bytes_ == rhs_obj.type_bytes_); + assert(lhs_obj.uncompressed_bytes() == rhs_obj.uncompressed_bytes()); + + std::vector lhs_buf(bytes); + std::vector rhs_buf(bytes); + + auto lhs_ds = lhs_obj.open(lhs_id); + auto rhs_ds = rhs_obj.open(rhs_id); + if (!lhs_ds || !rhs_ds) { + ++io_error_count; + continue; + } + + if (bytes <= max_buf_size) { + const bool lhs_read_ok = lhs_ds->read(lhs_buf); + const bool rhs_read_ok = rhs_ds->read(rhs_buf); + if (!lhs_read_ok || !rhs_read_ok) { + ++io_error_count; + continue; + } + + bytes_checked += bytes; + + if (lhs_buf != rhs_buf) { + show_diff_header(); + fmt::print( + ctx.msg(), "- {}: data / content diff:\n {}\n", + obj.name_, lhs_desc + ); + if (++diff_count >= topN) { + break; + } + } + } else { + // Warning: This ignores how the datasets are chunked. Could be slow... + std::vector dims(obj.rank_, 0); + for (size_t i = 0; i < obj.rank_; ++i) { + dims[i] = obj.dims_[i]; + } + auto div = etp12::AryDivision::make(dims, obj.type_bytes_, max_buf_size); + const size_t div_count = div.size(); + for (size_t i = 0; i < div_count; ++i) { + std::vector starts, counts; + const size_t offset = div.initDataSubarraysType(i, starts, counts); + + const bool lhs_read_ok = lhs_ds->read(lhs_buf, starts, counts); + const bool rhs_read_ok = rhs_ds->read(rhs_buf, starts, counts); + if (!lhs_read_ok || !rhs_read_ok) { + ++io_error_count; + break; + } + + const size_t read_bytes = etp12::productOf(counts) * div.esize_; + bytes_checked += read_bytes; + + if (lhs_buf != rhs_buf) { + show_diff_header(); + fmt::print( + ctx.msg(), "- {}: data / content diff (after {} bytes):\n {}\n", + obj.name_, thou(offset), lhs_desc + ); + if (++diff_count >= topN) { + break; + } + } + } + } + } + + if (diff_count) { + int diff_width = static_cast(std::log10(diff_count)); + diff_width += (diff_width / 3); // commas + diff_width += 1; + + fmt::print( + ctx.out(), "{}{} common datasets differ (out of {}).{}\n", + ctx.verbose? "\n": "", thou(diff_count), thou(match_count), + ctx.verbose? "": " (Use -v and --top to see which ones)" + ); + if (possible_int_conversion_diff == diff_count) { + fmt::print( + ctx.out(), + "ALL those differences could be harmless integer conversions:\n" + ); + } else if (possible_int_conversion_diff) { + fmt::print( + ctx.out(), + "{} of those differences could be harmless integer conversions:\n", + thou(possible_int_conversion_diff) + ); + } + + // Show the actual integer conversions that actually occurred + for (const auto& entry : int_convs) { + fmt::print( + ctx.out(), " {:>{}} {:>8} -> {:<8}\n", + thou(entry.second), diff_width, + hdf5NumTypeToString(entry.first.first), + hdf5NumTypeToString(entry.first.second) + ); + } + } else { + fmt::print( + ctx.out(), "No diffs in {} common datasets ({} uncompressed bytes).\n", + thou(match_count), thou(bytes_checked) + ); + } + + if (checked != match_count) { + fmt::print( + ctx.out(), "Warning: Stopped diff'ing after {} datasets ({:.1f}%)\n", + thou(checked), ((1. * checked) / match_count * 100.) + ); + } + + return true; +} + +} // namespace + +namespace oes::apps::eml { + +/****************************************************************************/ + +int hdf5_main(EmlAppContext& ctx, const UnparsedArgs& unparsed_args) { + po::options_description cmd_desc(R"( +Usage: openETPServer [global-options] hdf5 --info file.h5 +Usage: openETPServer [global-options] hdf5 --diff file1.h5 --with file2.h5 + +Actions)", ctx.terminalWidth_ + ); + + cmd_desc.add_options() + ("help,h", "Display this information") + + ("info", txtArg(), "Prints HDF5 file statistics") + ("stats", txtArg(), "Prints per-Dataset-type statistics") + ("diff", txtArg(), "Diffs two HDF5 files' dataset's metadata & bytes\n" + "(Ignores chunking, compression, attributes, etc...)") + ; + + po::options_description opt_desc("Options", ctx.terminalWidth_); + opt_desc.add_options() + ("top", intArg(10), "How many entry to list") + ("with", txtArg(), "Which HDF5 file to diff against") + ("max-buf-size,M", txtArg("1MB"), "Maximum buffer size in bytes") + ("force", "Continue instead of exit, on some warnings/errors") + ; + cmd_desc.add(opt_desc); + + if (int rc = ctx.reparse(cmd_desc, unparsed_args)) { + return (rc == -1)? 0: rc; + } + + EmlAppNamedAction actions[] { + { "info", &hdf5_info }, + { "stats", &hdf5_stats }, + { "diff", &hdf5_diff }, + }; + return ctx.dispatchActions(actions); +} + +} // namespace oes::apps::eml diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/ProbeCmds.cpp b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/ProbeCmds.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cbd66aaa79c98a85f3f9a8f67254cffd9787d46a --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/ProbeCmds.cpp @@ -0,0 +1,339 @@ +// ============================================================================ +// Copyright 2016-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "EmlAppContext.h" +#include "EtpClient12.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace oes::apps::eml; +using namespace oes::core; + +namespace po = boost::program_options; +namespace data12 = oes::eml::etp12::Datatypes; + +namespace { + +//============================================================================ + +bool alt_etp12_server_ping(const EmlAppContext& ctx) { + EtpClient12 clt(ctx); + if (!clt.connect()) { + return false; + } + + auto* proto = clt.core_proto(); + if (!proto) { + return false; + } + + auto rpc = etp::rpcFor(proto, &etp::proto12::ICore::Ping); + + using namespace std::chrono; + auto sys_now = time_point_cast(system_clock::now()); + rpc.req_.m_currentDateTime = sys_now.time_since_epoch().count(); + + auto before = time_point_cast(high_resolution_clock::now()); + auto rc = rpc.call(ctx.err(), clt.put_get_timeout_); + auto after = time_point_cast(high_resolution_clock::now()); + + if (!rc.first || rc.second != std::future_status::ready) { + ctx.err() << "Error: Ping failed" << std::endl; + return false; + } + + int64_t diff = rpc.res_.m_currentDateTime - rpc.req_.m_currentDateTime; + int64_t took = after.time_since_epoch().count() - before.time_since_epoch().count(); + int64_t skew = std::abs(diff - (std::max(2, took)/2)); + + microseconds server_us(rpc.res_.m_currentDateTime); + time_point server_tp(server_us); + + auto client_ts = formatExtendedIsoDateTime(sys_now, FracSecRes::eMicroSecFloor); + auto server_ts = formatExtendedIsoDateTime(server_tp, FracSecRes::eMicroSecFloor); + + ctx.out() << "Client Ping Timestamp: " << client_ts + << " UTC (" << thou(rpc.req_.m_currentDateTime) << "us since EPOCH)" << std::endl; + ctx.out() << "Server Pong Timestamp: " << server_ts + << " UTC (" << thou(rpc.res_.m_currentDateTime) << "us since EPOCH)" << std::endl; + if (diff < 0) { + ctx.out() << " Client-Server Diff: -" << thou(-diff) << "us" << std::endl; + } else { + ctx.out() << " Client-Server Diff: " << thou(diff) << "us" << std::endl; + } + ctx.out() << " Ping/Pong Roundtrip: " << thou(took) << "us" << std::endl; + ctx.out() << " Approx. Clock Skew: " << thou(skew) << "us" << std::endl; + + return true; +} + +//============================================================================ + +bool alt_etp12_server_info(const EmlAppContext& ctx) { + EtpClient12 clt(ctx); + if (!clt.config()) { + return false; + } + + std::string url = ctx.stringArg("server-url"); + if (url.empty()) { + ctx.err() << "Warning: No --server-url specified. Using localhost:9002" << std::endl; + url = "http://localhost:9002"; + } + + std::string req_url; + oes::core::HTTP req; + oes::core::HTTP::Response res; + std::vector headers{ "Accept: application/json" }; + + if (!clt.cfg_.auth_hdr_.empty()) { + headers.emplace_back("Authorization: " + clt.cfg_.auth_hdr_); + ctx.dbg() << "Authorization: " << clt.cfg_.auth_hdr_ << std::endl; + } + + req_url = url + "/.well-known/etp-server-capabilities?GetVersions=true"; + ctx.dbg() << "HTTP GET " << req_url << std::endl; + res = req.get(req_url, 0, headers); + if (res.code != 200) { + ctx.err() << "HTTP GET " << req_url << " KO: " + << res.code << ' ' << res.status << std::endl; + return false; + } + ctx.dbg() << "BODY (" << res.body.size() << " bytes):" << std::endl; + ctx.msg() << res.body << std::endl; + if (!json_validate_syntax(res.body.c_str(), res.body.size())) { + ctx.err() << "Error: Invalid JSON: GET ...?GetVersions=true" << std::endl; + return false; + } + std::vector versions; + if (!json_parse(res.body.c_str(), res.body.size(), versions, JSON_TYPE_STRING)) { + ctx.err() << "Error: Unexpected JSON: GET ...?GetVersions=true" << std::endl; + return false; + } + auto& out = ctx.out(); + + switch (const size_t N = versions.size()) { + case 0: out << "Warning: Server advertises NO versions." << std::endl; break; + case 1: out << "Server advertises 1 version:" << std::endl; break; + default: out << "Server advertises " << N << " versions:" << std::endl; break; + } + + for (const std::string& v : versions) { + out << "- " << v << std::endl; + } + + for (const std::string& v : versions) { + res.body.clear(); + res.status.clear(); + out << "\n=== " << v << " ===" << std::endl; + req_url = url + "/.well-known/etp-server-capabilities?GetVersion=" + v; + ctx.dbg() << "HTTP GET " << req_url << std::endl; + res = req.get(req_url, 0, headers); + if (res.code != 200) { + ctx.err() << "HTTP GET " << req_url << " KO: " + << res.code << ' ' << res.status << std::endl; + continue; + } + ctx.dbg() << "BODY (" << res.body.size() << " bytes):" << std::endl; + ctx.msg() << res.body << std::endl; + if (!json_validate_syntax(res.body.c_str(), res.body.size())) { + ctx.err() << "Error: Invalid JSON: GET ...?GetVersion=" << v << std::endl; + continue; + } + data12::ServerCapabilities caps; + oes::eml::my_buffer_ptr buf(new std::vector( + res.body.begin(), res.body.end() + )); + try { + oes::eml::decode_json(caps, buf); + } catch (const std::exception& ex) { + ctx.err() << "Error: Avro JSON Parse: " << ex.what() << std::endl; + ctx.err() << "\tof GET ...?GetVersion=" << v << std::endl; + continue; + } catch (...) { + ctx.err() << "Error: Avro JSON Parse: GET ...?GetVersion=" << v << std::endl; + continue; + } + + fmt::print( + out, "{} ({})\n{}\n\n", + caps.m_applicationName, + caps.m_applicationVersion, + caps.m_contactInformation.m_organizationName + ); + + switch (const size_t N = caps.m_supportedProtocols.size()) { + case 0: out << "Warning: NO protocols. Ignored!" << std::endl; continue; + case 1: out << "Warning: Just 1 protocol:" << std::endl; break; + default: out << N << " protocols:" << std::endl; break; + } + for (const auto& proto : caps.m_supportedProtocols) { + using namespace oes::eml::etp12; + auto found = protocols().find(proto.m_protocol); + if (found == protocols().end()) { + out << "Warning: Unknown protocol ID: " << proto.m_protocol << std::endl; + continue; + } + const ProtocolInfo& info = found->second; + + out << "- " << info.label() << std::endl; + } + + const size_t nEnc = caps.m_supportedEncodings.size(); + const size_t nCmp = caps.m_supportedCompression.size(); + const size_t nFmt = caps.m_supportedFormats.size(); + + if (nEnc + nCmp + nFmt) { + out << std::endl; + } + + if (nEnc) { + std::string csv = boost::join(caps.m_supportedEncodings, ", "); + out << nEnc << " supported encodings: " << csv << std::endl; + } + if (nCmp) { + std::string csv = boost::join(caps.m_supportedCompression, ", "); + out << nCmp << " supported compression: " << csv << std::endl; + } + if (nFmt) { + std::string csv = boost::join(caps.m_supportedFormats, ", "); + out << nFmt << " supported formats: " << csv << std::endl; + } + + if (const size_t N = caps.m_supportedDataObjects.size()) { + out << "\n" << N << " supported objects: " << std::endl; + for (const auto& type : caps.m_supportedDataObjects) { + auto& obj_caps = type.m_dataObjectCapabilities; + + // keys of obj_caps are supposed to be case-insensitive, + // so we must normalize the keys first... + std::map imap; + for (auto it = obj_caps.begin(); it != obj_caps.end(); ++it) { + imap.emplace(boost::to_lower_copy(it->first), &it->second); + } + + std::vector verb_caps; + auto it = obj_caps.find("supportsget"); + if (it != obj_caps.end()) { + if (const bool* p_get = std::get_if(&it->second.m_item)) { + if (*p_get) { + verb_caps.emplace_back("get"); + } + } + } + it = obj_caps.find("supportsput"); + if (it != obj_caps.end()) { + if (const bool* p_put = std::get_if(&it->second.m_item)) { + if (*p_put) { + verb_caps.emplace_back("put"); + } + } + } + it = obj_caps.find("supportsdelete"); + if (it != obj_caps.end()) { + if (const bool* p_del = std::get_if(&it->second.m_item)) { + if (*p_del) { + verb_caps.emplace_back("delete"); + } + } + } + // TODO: Show other caps, from DataObjectCapabilityKind + std::string csv = boost::join(verb_caps, ", "); + fmt::print(out, "- {} ({})\n", type.m_qualifiedType, !csv.empty()? csv: "no caps"); + } + } + + auto max_msg_found = caps.m_endpointCapabilities.find("MaxWebSocketMessagePayloadSize"); + // FIXME: Remove: Temporary work-around for PV's server + if (max_msg_found == caps.m_endpointCapabilities.end()) { + max_msg_found = caps.m_endpointCapabilities.find("maxWebSocketMessagePayloadSize"); + } + if (max_msg_found != caps.m_endpointCapabilities.end()) { + size_t N = 0; + const data12::DataValue_item_v& val = max_msg_found->second.m_item; + switch (val.index()) { + case size_t(data12::DataValue_item_e::eDouble): N = size_t(std::get(val)); break; + case size_t(data12::DataValue_item_e::eFloat): N = size_t(std::get(val)); break; + case size_t(data12::DataValue_item_e::eLong): N = size_t(std::get(val)); break; + case size_t(data12::DataValue_item_e::eInt): N = size_t(std::get(val)); break; + default: + out << "\nWarning: Unknown type for MaxWebSocketMessagePayloadSize value" << std::endl; + } + if (N) { + out << "\nMaximum WebSocket Message Payload Size: " << thou(N) << " bytes" << std::endl; + } + } else { + out << "\nNO Maximum WebSocket Message Payload Size advertised!" << std::endl; + } + } + + return true; +} + +//============================================================================ + +} //namespace + +namespace oes { +namespace apps { +namespace eml { + +/****************************************************************************/ + +int probe_main(EmlAppContext& ctx, const UnparsedArgs& unparsed_args) { + po::options_description cmd_desc( + "\nUsage: openETPServer [global-options] " + ctx.command + " --info -S URL\n" + "Usage: openETPServer [global-options] " + ctx.command + " --ping -S URL [auth-options]\n" + "\nActions", ctx.terminalWidth_ + ); + cmd_desc.add_options() + ("help,h", "Display this information") + + ("info", "Get ETP 1.2 server capabilities") + ("ping", "Connect and ping ETP 1.2 server") + ; + + cmd_desc.add(connect_options(ctx.terminalWidth_)); + cmd_desc.add(authenticate_options(ctx.terminalWidth_)); + + if (int rc = ctx.reparse(cmd_desc, po::positional_options_description(), unparsed_args)) { + return (rc == -1)? 0: rc; + } + + EmlAppNamedAction actions[] { + { "info", &alt_etp12_server_info }, + { "ping", &alt_etp12_server_ping }, + }; + return ctx.dispatchActions(actions); +} + +} // namespace eml +} // namespace apps +} // namespace oes diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/ServerCmds.cpp b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/ServerCmds.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a33d808b90f72e1a492f306acbfc2ce9facb9de3 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/ServerCmds.cpp @@ -0,0 +1,440 @@ +// ============================================================================ +// Copyright 2016-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "EmlAppContext.h" +#include "EtpClient12.h" + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +using namespace oes::apps::eml; +using namespace oes::core; +using namespace oes::eml; + +namespace fs = std::filesystem; +namespace po = boost::program_options; + +using oes::eml::etp12::EmlUri; + +namespace data12 = oes::eml::etp12::Datatypes; +using Etp12Resources = std::vector; + +namespace { + +//============================================================================ + +struct MainRepo { + bool ok_; + + openkv::Connection_ptr conn_; + openkv::OpenKVAdminDB_ptr admin_db_; + + fs::path admin_dir_; + fs::path admin_dbfile_; + + MainRepo(bool ok = false) : ok_(ok), admin_db_(nullptr) {} + + explicit operator bool() const { return ok_; } +}; + +MainRepo getMainRepo( + const EmlAppContext& ctx, bool init +) { + std::string dir = ctx.stringArg("db-connection"); + if (dir.empty()) { + const char* connection_string = std::getenv("POSTGRESQL_CONN_STRING"); + if(connection_string!=nullptr){ + dir = std::string(connection_string); + } + if (dir.empty()) { + ctx.err() << "Error: Missing required --db-connection or POSTGRESQL_CONN_STRING environment variable" << std::endl; + return MainRepo(); + } + } + + const std::string dbfile = dir; + OES_UNUSED(init); + + MainRepo main; + main.conn_.reset(new oes::emldb::Connection(dbfile.c_str())); + main.admin_dir_ = dbfile; + main.admin_dbfile_ = dbfile; + main.admin_db_ = std::make_shared(main.conn_, main.admin_dbfile_.string()); + main.ok_ = true; + + return main; +} + +//============================================================================ + +bool alt_etp12_openkv_start(const EmlAppContext& ctx) { + MainRepo au = getMainRepo(ctx, false); + if (!au) { + return false; + } + + std::ostringstream oss; + oss << "SELECT set_config('application_name','" << au.admin_db_->app_id() << "', false);"; + const std::string app_pragmas = oss.str(); + au.admin_db_->handle()->execute(app_pragmas.c_str()); + if(!au.admin_db_->isSchemaCreated()){ + au.admin_db_->createSchema(); + } + if (!au.admin_db_->isSchemaUpToDate(false)) { + ctx.err() << "Error: Cannot start server: MainRepo schema out-of-date." << std::endl; + return false; // no upgrades for the time being... + } + + //if (ctx.dataspaceSpecified()) { + // ctx.out() << "Serving Dataspace " << ctx.dataspaceTitle() << std::endl; + //} else { + // ctx.out() << "Serving All Dataspaces" << std::endl; + //} + + std::string jwtSecret = ctx.stringArg("jwt-secret"); + + std::string authN = ctx.stringArg("authN"); + if (ctx.hasArg("auth")) { + if (ctx.hasArg("authN") && !ctx.args["authN"].defaulted()) { + ctx.err() << "Error: --auth replaced by --authN. Please use the latter." << std::endl; + return false; + } else { + ctx.out() << "Warning: --auth deprecated: Please use --authN instead." << std::endl; + authN = ctx.stringArg("auth"); + } + } + + if (authN != "none" && authN != "builtin" && !boost::starts_with(authN, "delegate=")) { + ctx.err() << "Error: Invalid --authN: Use 'none', 'builtin', or 'delegate=URL'" << std::endl; + return false; + } + + const std::string authZ = ctx.stringArg("authZ"); + if (authZ != "none" && authZ != "builtin" && !boost::starts_with(authZ, "delegate=")) { + ctx.err() << "Error: Invalid --authZ: Use 'none', 'builtin', or 'delegate=URL'" << std::endl; + return false; + } + + if (authN != "builtin" && !jwtSecret.empty()) { + ctx.out() << "Warning: Option --jwt-secret ignored; requires --authN=builtin." << std::endl; + } + + // AuthN, i.e. Authentication + openkv::OpenKVAuthNMgr authN_mgr; + if (authN == "none") { + ctx.out() << "Warning: Authentication disabled." << std::endl; + authN_mgr = [&ctx]( + const std::string& auth_hdr, auth::AuthDetails* p_details + ) { + auth::AuthenticatedUser au; + + // try to get the current user, for AuthZ later. + if (!auth_hdr.empty()) { + ctx.out() << "Warning: Authentication header ignored: --authN none" << std::endl; + + // We try decoding the header with a dummy JWT secret, that will fail + // of course, but give us the jwt_payload, to reparse that as JSON, and + // we look for the "username" key, i.e. the same "claim" as our JWT processing. + auth::AuthDetails local_details; + auth::AuthDetails& details = p_details? *p_details: local_details; + au = auth::AuthenticatedUser::decodeFromHeader( + auth_hdr, "jwt-secret", &details + ); + auth::AuthType prev_type = au.type_; + + if (!au.ok_ && !details.jwt_payload_.empty()) { + au = auth::AuthenticatedUser::decodeFromJson( + details.jwt_payload_, false, &details + ); + au.type_ = prev_type; // otherwise eUnknown overrides eBearer + } + } + if (au.name_.empty()) { + au.name_ = "nobody"; // failed to get username from auth_hdr + } + au.since_ = std::chrono::system_clock::now(); + au.read_only_ = false; // No authN, leave it to authZ (if any) + au.header_ = auth_hdr; + au.ok_ = true; + return au; + }; + } else if (authN == "builtin") { + if (jwtSecret.empty()) { + ctx.out() << "Warning: No --jwt-secret: Bearer authentication disabled (Basic only)." << std::endl; + } + + // TODO: Use MainRepo's users table, instead of hard-coded names below + std::string username = "foo"; + std::string password = "bar"; + openkv::OpenKVUserMgr user_mgr = + [usr = username, pwd = password] + (const std::string& username, const std::string& password) { + return (username == usr) && (password == pwd); + } + ; + authN_mgr = openkv::builtinAuthMgr(user_mgr, jwtSecret); + } else { + assert(false); + return false; + } + + // TODO: AuthZ, i.e. Authorization + if (authZ == "none") { + if (!ctx.args["authZ"].defaulted()) { + ctx.out() << "Warning: Authorization disabled." << std::endl; + } + } else if (authZ == "builtin") { + ctx.out() << "Warning: No built-in authorization yet." << std::endl; + } else if (boost::starts_with(authZ, "delegate=")) { + std::string url = authZ.substr(9); + ctx.err() << "Error: Delegate authorization to URL endpoint " + << url << ": TODO" << std::endl; + return false; + } else { + assert(false); + return false; + } + + openkv::OpenKVSpaceResolver space_resolver = + [admin_dir = au.admin_dir_] + (const std::string& relative_dbfile, bool new_db) -> std::string { + // See also how new_space.dbfile_ is assigned in this file, + // to be relative to main-folder *and* using "unix path" + OES_UNUSED(new_db); + return relative_dbfile; + } + ; + + oes::eml::etp12::ServerConfig config = openkv::makeServerConfig( + au.admin_db_, std::move(authN_mgr), nullptr, std::move(space_resolver) + ); + + config.port_ = ctx.args["port"].as(); + config.reuseAddressSocket_ = ctx.args["reuseAddressSocket"].as(); + config.stopOnLastDisconnection_ = ctx.args["stopOnLastDisconnection"].as(); + config.useIpV4_ = ctx.args["ipv4"].as(); + config.maxNumberConnection_ = ctx.args["maxConnections"].as(); + + config.verbose_ = ctx.verbose; + config.debug_ = ctx.debug; + config.quiet_ = ctx.quiet; + + const std::string& mms_spec = ctx.stringArg("max-msg-size"); + if (!mms_spec.empty()) try { + auto [zsize, usize] = parseByteSizes(mms_spec); + config.max_mgs_size_ = zsize; + config.max_mgs_usize_ = usize; // Warning: Not ETP standard + } catch (const std::runtime_error& ex) { + ctx.err() << "Error: Invalid --max-msg-size option: " << ex.what() << std::endl; + return false; + } + + // Warning: NOT ETP standard. Used to exist, but was eventually removed, in favor + // of purely max_mgs_size-driven splitting of messages into multi-part responses. + const int max_batch_size = ctx.args["max-batch-size"].as(); + if (!(0 <= max_batch_size && max_batch_size <= 10'000) && (max_batch_size != 1'000'000)) { + ctx.err() << "Error: Invalid --max-batch-size: Not in [0 - 10,000] range" << std::endl; + return false; + } + config.max_batch_size_ = max_batch_size; + + const int mqd = ctx.args["msg-queue-depth"].as(); + if (!(0 < mqd && mqd <= 100)) { + ctx.err() << "Error: Invalid --msg-queue-depth option: Not in range [1, 100]" << std::endl; + return false; + } + config.msg_queue_depth_ = mqd; + + if (ctx.hasArg("jobs")) { + // single-threaded when -j is not explicitly specified + int64_t jobs = ctx.args["jobs"].as(); + OES_ASSERT(jobs >= 0, jobs = 1;); + config.totalThreadCount_ = static_cast(jobs); + } + + if (ctx.hasArg("etp-trace")) { + config.enable_tracing_ = true; + config.trace_file_ = ctx.stringArg("etp-trace"); + } + + oes::eml::etp12::Server server(config); + data12::ServerCapabilities svr_info = server.getServerInfo(); + ctx.out() << "" << svr_info.m_applicationName << ' ' << svr_info.m_applicationVersion << std::endl; + return server.start(); +} + +//============================================================================ + +bool alt_etp12_openkv_init_admin_db(const EmlAppContext& ctx) { + MainRepo au = getMainRepo(ctx, false); + if (!au) { + return false; + } + + std::ostringstream oss; + oss << "SELECT set_config('application_name','" << au.admin_db_->app_id() << "', false);"; + const std::string app_pragmas = oss.str(); + au.admin_db_->handle()->execute(app_pragmas.c_str()); + + const bool ok = au.admin_db_->createSchema(); + if (ok) { + ctx.out() << "Created main DB" << std::endl; + } else { + ctx.err() << "Error: Cannot create main DB" << std::endl; + } + + // Create the ETP 1.2 mandatory "default" (no name) Dataspace + if (ok && !ctx.hasArg("no-default-space")) { + // TODO: Create default space + } + + return ok; +} + +bool alt_etp12_openkv_check_admin_db(const EmlAppContext& ctx) { + MainRepo au = getMainRepo(ctx, false); + if (!au) { + return false; + } + + const bool created = au.admin_db_->isSchemaCreated(); + if (!created) { + ctx.err() << "Error: Empty main DB: " << au.admin_dbfile_ << std::endl; + return false; + } + const bool uptodate = au.admin_db_->isSchemaUpToDate(false); + if (!uptodate) { + ctx.out() << "Warning: Out-of-date main DB: " << au.admin_dbfile_ << std::endl; + return true; + } + ctx.out() << "OK. Up-to-date main DB: " << au.admin_dbfile_ << std::endl; + return true; +} + +//============================================================================ + +} //namespace + +namespace oes::apps::eml { + +/****************************************************************************/ + +int server_main(EmlAppContext& ctx, const UnparsedArgs& unparsed_args) { + po::options_description cmd_desc( + "\nUsage: openETPServer [global-options] " + ctx.command + " --start -d conn_string [options]\n" + "\nActions", ctx.terminalWidth_ + ); + cmd_desc.add_options() + ("help,h", "Display this information") + ("init", "Initialize main DB") + ("check", "Check main DB") + ("start", "Start ETP 1.2 server") + ; + + po::options_description opt_desc("Options", ctx.terminalWidth_); + opt_desc.add_options() + ("port,P", + po::value()->default_value(9002), + "Specify port to listen to") + ("reuseAddressSocket", + po::value()->default_value(true), + "Reuse socket address. Allows reusing socket port\n" + "shortly after previous server instance stopped.") + ("stopOnLastDisconnection", + po::value()->default_value(true), + "Stop when last client disconnect") + ("ipv4", + po::value()->default_value(false), + "Use ipv4 protocol instead of ipv6") + ("maxConnections", + po::value()->default_value(-1), + "Specify maximum number of connections") + + ("db-connection,d", txtArg(), + "PostgreSQL connection string (If not defined, will use POSTGRESQL_CONN_STRING environment variable") + ("space-path,s", txtArg(), "ETP dataspace name (folder or folder/project)") + ("space-guid", txtArgOpt(), "ETP dataspace guid (uses path if omitted)") + + ("xdata-file", txtArg(), "New Space meta-data as JSON file (on EPC Direct-import)") + ("vacuum", "Vacuum (i.e. optimize) DB, post EPC Direct-import") + + ("no-default-space", "Omit ETP 1.2 mandatory 'default' dataspace on --init") + + ("overwrite", "Overwrite existing EPC on Direct-import") + + // Auth-related options + ("auth", txtArg(), "Deprecated: Legacy name of authN") + ("authN,N", txtArg("builtin"), "Authentication: 'none', 'builtin', or 'delegate=URL'") + ("authZ", txtArg("none"), "Authorization: 'none', 'builtin', or 'delegate=URL'") + ("jwt-secret,k", txtArg(), "JWT Signing Key (--authN=builtin only).") + + ("jobs,j", + greedy_value()->implicit_value(0), + "How many worker-threads:\n" + "* not specified: Defaults to single-threaded;\n" + "* -j: Use as many threads as hardware cores;\n" + "* -jN: Use exactly N threads") + + ("force", "Continue instead of exit, on some warnings/errors") + ; + + EtpDefaults defaults; + defaults.max_msg_size_ = "16MB, 128MB"; + defaults.max_batch_size_ = 1'000'000; // I.e. disable it for now, by default + defaults.compress_ = true; + + cmd_desc.add(opt_desc); + cmd_desc.add(connect_options(ctx.terminalWidth_, defaults)); + + if (int rc = ctx.reparse(cmd_desc, po::positional_options_description(), unparsed_args)) { + return (rc == -1)? 0: rc; + } + + EmlAppNamedAction actions[] { + { "init", &alt_etp12_openkv_init_admin_db }, + { "check", &alt_etp12_openkv_check_admin_db }, + { "start", &alt_etp12_openkv_start }, + }; + return ctx.dispatchActions(actions); +} + +} // namespace oes::apps::eml diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/SpaceCmds.cpp b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/SpaceCmds.cpp new file mode 100644 index 0000000000000000000000000000000000000000..439e7ac56e1ae51448bf44eb3f1781462502377b --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/SpaceCmds.cpp @@ -0,0 +1,1591 @@ +// ============================================================================ +// Copyright 2016-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "EmlAppContext.h" +#include "EtpClient12.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace oes::apps::eml; + +namespace fs = std::filesystem; +namespace po = boost::program_options; +namespace etp = oes::eml::etp12; + +namespace { + +using oes::core::thou; +using oes::eml::etp12::rpcFor; +const std::string dflt = ""; + +int64_t parseSince(const std::string& since, bool utc) { + if (since.empty()) { + return 0; + } + + // Not a valid ISO date-time + using namespace std::chrono; + microseconds since_epoch(0); // UTC or Local, it depends + if (oes::core::usecSinceEpochFromExtendedIsoDateTime(since, since_epoch)) { + if (!utc) { + // We assume the date we parsed is in local time, not UTC + /* + ** This requires date/tz.h or C++20 + time_point tp_localtime(since_epoch); + auto local_time = date::make_zoned(date::current_zone(), tp_localtime); + return local_time.get_sys_time().time_since_epoch().count(); + */ + system_clock::time_point tp(since_epoch); + tp = oes::core::localToUtc(tp); + system_clock::duration ts = tp.time_since_epoch(); + since_epoch = duration_cast(ts); + } + return since_epoch.count(); + } + + std::istringstream iss(since); + int64_t num; + std::string uom; + iss >> num >> uom; + if (iss.fail() || !iss.eof()) { + return -1; + } + + microseconds offset(0); + switch (uom.size()) { + case 0: return -1; + case 1: + switch (uom[0]) { + case 's': offset = seconds(num); break; + case 'h': offset = hours(num); break; + // days, weeks, months, years are C++20 only + case 'd': offset = duration>(num); break; + case 'w': offset = duration>(num); break; + case 'm': offset = duration>(num); break; + case 'y': offset = duration>(num); break; + } + break; + case 2: return -1; + case 3: + if (uom == "min") { + offset = minutes(num); + } + break; + default: return -1; + } + + if (offset.count() == 0) { + return -1; + } + + const auto now = system_clock::now(); // UTC + const auto since_usec = time_point_cast(now - offset); + return since_usec.time_since_epoch().count(); +} + +std::string printMicroSecDateTime(int64_t etp_usec, bool utc, bool iso, bool debug) { + using namespace oes::core; + using namespace std::chrono; + microseconds usec(etp_usec); // UTC + time_point tp(usec); + if (!utc) { + tp = time_point_cast(utcToLocal(tp)); + } + std::string dt_fmt = iso? isoDatetimeFormat(): ascDatetimeFormat(); + FracSecRes dt_res = debug? FracSecRes::eMicroSecFloor: FracSecRes::eWholeSecFloor; + std::string ts = formatDateTime(dt_fmt, tp, dt_res); + if (utc) { + ts += " UTC"; + } + return ts; +} + +std::pair getSpacePath( + const EmlAppContext& ctx, + bool required = false, + bool default_ok = true +) { + if (!ctx.hasArg("space-path")) { + if (required) { + ctx.err() << "Error: Missing --space-path option" << std::endl; + return { {}, false }; + } + } + + std::string space_path = ctx.stringArg("space-path"); + if (space_path.empty()) { + if (!default_ok && !ctx.hasArg("force")) { + ctx.err() << "Error: Cannot use " << dflt << " unnamed dataspace" << std::endl; + return { {}, false }; + } + ctx.out() << "Warning: Using " << dflt << " unnamed dataspace" << std::endl; + } + return { std::move(space_path), true }; +} + +//============================================================================ + +// Convert special (Windows) chars that cannot appear in filenames +// Linux accepts arbitrary strings for filenames, but lets be consistent. +constexpr char win_special_char[] = "\\/:*?\"<>|"; +std::string scrubFilename(const std::string& s) { + if (s.find_first_of(win_special_char) == std::string::npos) { + return s; + } + std::string copy(s); + for (char& c : copy) { + if (std::strchr(win_special_char, c)) { + c = '#'; + } + } + return copy; +}; + +bool tryRemovefile(std::ostream& err, const std::string& filename, bool overwrite) { + if (fs::exists(filename)) { + if (!overwrite) { + err << "Error: Output file exists (and --overwrite not specified)" << std::endl; + return false; + } + if (!fs::remove(filename)) { + err << "Error: Output file exists and cannot be removed" << std::endl; + return false; + } + } + return true; +} + +// Unfortunately, fs::status(path).permissions() is not reliable on Windows. +// Especially for folder in C:\Program Files, with special UAC protections. +bool canWriteFileInFolder(const std::string& filename) { + namespace fs = fs; + fs::path path(filename); + const size_t len = path.filename().string().size(); + + // Warning: .parent_path() of relation path may be empty, + // must convert to absolute path before getting its parent. + fs::path folder = absolute(path).parent_path(); + if (!is_directory(folder)) { + return false; + } + + bool can_write = false; + for (int i = 0; i < 7; ++i) { + // to avoid running in path length limitations, especially + // on Windows, don't use the full 32 chars of the Guid, but + // as many of the input file name. But Use a minimum of 7 chars, + // just in case, and account for those pre-existing (100 tries). + std::string uid = oes::core::Guid::make().toStringNoDash(); + std::string stem = 'x' + uid.substr(0, std::max(7, len)); + fs::path tmp_file = folder / stem; + if (exists(tmp_file)) { + continue; + } + try { + std::ofstream ofs(tmp_file.string()); + if (ofs.is_open()) { + ofs << '\n'; + ofs.close(); + can_write = true; + try { + fs::remove(tmp_file); + } catch (const std::exception&) { + // ignore, i.e. leave temp file behind... + // If we do not catch here, we don't break below + } + break; + } + } catch (const std::exception&) { + // ignore? or break? We continue spinning if ignored, + // but only 7 times, so lets give that a try. + } + } + return can_write; +} + +//============================================================================ + +int hdf5Level(const EmlAppContext& ctx, const std::string& opt_name) { + // we provide a default + OES_DEBUG_ASSERT(ctx.hasArg(opt_name)); + + const int zlevel = ctx.args[opt_name].as(); + if (0 <= zlevel && zlevel <= 9) { + return zlevel; + } + fmt::print( + ctx.err(), + "Warning: Invalid --{} level {}. Not in [0-9] range. Assuming 0.\n", + opt_name , zlevel + ); + return 0; // not compressed +} + +int hdf5CompressionLevel(const EmlAppContext& ctx) { + return hdf5Level(ctx, "hdf5-compression"); +} + +int hdf5LargeDataCompressionLevel(const EmlAppContext& ctx) { + return hdf5Level(ctx, "hdf5-large-data-compression"); +} + +//============================================================================ + +bool space_new(const EmlAppContext& ctx) { + auto [space_path, space_ok] = getSpacePath(ctx, true, false); + if (!space_ok) { + return false; + } + + oes::eml::etp12::XDataMap xdata; + std::string xdata_file = ctx.stringArg("xdata-file"); + if (!xdata_file.empty()) { + if (!oes::eml::etp12::xdataFromJsonFile(xdata, xdata_file)) { + ctx.err() << "Error: Cannot read or parse JSON file: " << xdata_file << std::endl; + return false; + } + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDataspace + }); + if (!clt.connect()) { + return false; + } + + auto* proto = clt.space_proto(); + if (!proto) { + return false; + } + + auto rpc = rpcFor(proto, &etp::proto12::IDataspace::PutDataspaces); + + etp::data12::Object::Dataspace space; + space.m_path = space_path; + if (!xdata.empty()) { + space.m_customData.swap(xdata); + ctx.dbg() << "Added " << space.m_customData.size() + << " custom-data from " << xdata_file << std::endl; + } + + // by default, the URI is derived from the path, + // as per the discussion on the ETP forum. + std::string uid = space_path; + + // But as an extension, we allow using a Guid-based URI instead. + // And the client can now control the Guid that will be used. + // Unless one is not explicitly provided, in which case we make it up. + const bool use_guid = ctx.hasArg("space-guid"); + if (use_guid) { + using oes::core::Guid; + std::string guid_txt = ctx.stringArg("space-guid"); + Guid guid = guid_txt.empty()? Guid::make(): Guid::parse(guid_txt); + if (!guid.isValid()) { + ctx.err() << "Error: Invalid UUID for --space-guid: " << guid_txt << std::endl; + return false; + } + uid = guid.toXmlString(); + } + space.m_uri = etp::EmlUri::make_space(uid).to_string(); + + rpc.req_.m_dataspaces.emplace("1", space); + + auto ok = rpc.call(ctx.err(), clt.put_get_timeout_); + if (!ok.first) { + return false; + } + + return true; +} + +bool space_info(const EmlAppContext& ctx) { + auto [space_path, space_ok] = getSpacePath(ctx); + if (!space_ok) { + return false; + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDataspace + }); + if (!clt.connect()) { + return false; + } + + auto* proto = clt.space_proto(); + if (!proto) { + return false; + } + + auto get_rpc = rpcFor(proto, &etp::proto12::IDataspace::GetDataspaces); + auto get_ok = get_rpc.call(ctx.err(), clt.put_get_timeout_); + if (!get_ok.first) { + return false; + } + + const auto& spaces = get_rpc.res_.m_dataspaces; + auto found = std::find_if( + spaces.begin(), spaces.end(), [space_path](const auto& space) { + return space.m_path == space_path; + } + ); + if (found == spaces.end()) { + ctx.err() << "Error: ETP Dataspace not found: " << space_path << std::endl; + return false; + } + const auto& space = *found; + + auto uri = etp::EmlUri::parse(space.m_uri); + if (uri.type_ != etp::EmlUri::eDATA_SPACE) { + ctx.err() << "Warning: Invalid URI: Not of dataspace type: " << space.m_uri << std::endl; + } + + int64_t cetp_usec = space.m_storeCreated; + std::string cts = printMicroSecDateTime( + cetp_usec, ctx.hasArg("utc"), ctx.hasArg("iso"), ctx.debug + ); + + int64_t metp_usec = space.m_storeLastWrite; + std::string mts = printMicroSecDateTime( + metp_usec, ctx.hasArg("utc"), ctx.hasArg("iso"), ctx.debug + ); + + std::ostream& out = ctx.out(); + fmt::print(out, R"( +Dataspace {}: + URI: {} + Uid: {} + Path: {} + Created: {} + Last-Modified: {} +)", + space_path.empty()? dflt: space_path, + space.m_uri, uri.dataspace_, space.m_path, cts, mts + ); + + if (const size_t N = space.m_customData.size()) { + fmt::print(out, "\n{} custom data:\n", N); + size_t max_key = 0; + for (const auto& entry : space.m_customData) { + const size_t ks = entry.first.size(); + if (ks <= 32u) { + max_key = std::max<>(max_key, ks); + } + } + const size_t tw = ctx.terminalWidth_; + const size_t kw = std::max(14, std::min(24, max_key)); // key width + const size_t vw = std::max(tw, 48) - kw - 3; // value width + using DV = etp::Datatypes::DataValue_item_e; + for (const auto& entry : space.m_customData) { + const auto& key = entry.first; + if (key.size() <= kw) { + fmt::print(out, "{:>{}}: ", key, kw); + } else { + fmt::print(out, "{:>{}}:\n{:<{}} ", key, kw, "", kw); + } + + const auto& item = entry.second.m_item; + switch (auto idx = item.index()) { + case size_t(DV::eNull): fmt::print(out, "\n"); break; + case size_t(DV::eBool): fmt::print(out, "{}\n", std::get(item)); break; + case size_t(DV::eInt): fmt::print(out, "{}\n", std::get(item)); break; + case size_t(DV::eLong): fmt::print(out, "{}\n", std::get(item)); break; + case size_t(DV::eFloat): fmt::print(out, "{}\n", std::get(item)); break; + case size_t(DV::eDouble): fmt::print(out, "{}\n", std::get(item)); break; + case size_t(DV::eString): { + const std::string& val = std::get(item); + const auto line = val.substr(0, vw); + fmt::print(out, "{}\n", line); + if (val.size() > vw) { + fmt::print( + out, "{:<{}} (and {} more bytes, truncated...)\n", + "", kw, (val.size() - vw) + ); + } + break; } + case size_t(DV::eBytes): { + const auto& val = std::get>(item); + fmt::print(out, "\n", val.size()); + break; } + default: + fmt::print(out, "TODO: Print array value (type #{})\n", idx); + } + } + } else { + fmt::print(out, "\nNo custom data.\n"); + } + + return true; +} + +bool space_list(const EmlAppContext& ctx) { + const bool iso = ctx.hasArg("iso"); + const bool utc = ctx.hasArg("utc"); + const std::string since = ctx.stringArg("since"); + const int64_t lastChangedFilter = parseSince(since, utc); + if (lastChangedFilter < 0) { + ctx.err() << "Error:\tInvalid --since value: " << since << + "\n\tUse an offset: 5min, 1d, 1w; or an ISO-8601 date-time (see --iso)"<< std::endl; + return false; + } else if (ctx.verbose && lastChangedFilter != 0) { + using namespace std::chrono; + microseconds usec(lastChangedFilter); // UTC + system_clock::time_point ts(usec); + ctx.msg() << "Dataspaces since " << oes::core::formatIsoDateTime(ts) << " UTC" << std::endl; + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDataspace + }); + if (!clt.connect()) { + return false; // error already issued + } + + size_t total = 0; + const std::string pattern = ctx.stringArg(ctx.action()); + auto spaces = clt.globSpacesSince(pattern, lastChangedFilter, &total); + + auto print = [&ctx, utc, iso](const etp::Datatypes::Object::Dataspace& space, size_t max_len) { + const std::string& space_path = space.m_path.empty()? dflt: space.m_path; + max_len = max_len? max_len: space_path.length(); + + int64_t etp_usec = space.m_storeLastWrite; + std::string ts = printMicroSecDateTime(etp_usec, utc, iso, ctx.debug); + + bool ro = false; + auto ro_found = space.m_customData.find("read-only"); + if (ro_found != space.m_customData.end()) { + const auto& ro_entry = ro_found->second.m_item; + if (const bool* p_ro = std::get_if(&ro_entry)) { + ro = *p_ro; + } + } + + const size_t n = space.m_customData.size(); + std::string xdata = !n? "": fmt::format(" ({} custom data)", n); + + ctx.out() << fmt::format( + "- {:<{}} {} {}{}", + space_path, max_len, (ro? "r-": "rw"), ts, xdata + ) << std::endl; + }; + + const bool flat = ctx.hasArg("flat"); + using SpacePtrs = std::vector; + using GroupedSpaceMap = std::map; + GroupedSpaceMap spaces_by_folders; + SpacePtrs all_spaces; // kinda, already filtered in fact + for (const auto& space : spaces) { + all_spaces.emplace_back(&space); + if (!flat) { + const size_t slash = space.m_path.find_last_of('/'); + if (!flat && slash != std::string::npos) { + std::string folder = space.m_path.substr(0, slash); + spaces_by_folders[folder].emplace_back(&space); + } else { + spaces_by_folders[{}].emplace_back(&space); + } + } + } + + auto print_spaces = [&print]( + std::ostream& os, const auto& spaces, const std::string& suffix + ) { + size_t max_len = 0; + switch (auto N = spaces.size()) { + case 0: os << "No dataspaces" << std::endl; break; + case 1: os << "1 dataspace" << suffix << ":\n"; print(*spaces[0], 0); break; + default: os << thou(N) << " dataspaces" << suffix << ":\n"; + for (const auto& p_space : spaces) { + const auto& p = p_space->m_path; + max_len = std::max<>(max_len, p.empty()? dflt.size(): p.size()); + } + for (const auto& p_space : spaces) { + print(*p_space, max_len); + } + } + }; + + if (flat) { + print_spaces(ctx.out(), all_spaces, ""); + } else { + const etp::data12::Object::Dataspace* dflt_space = nullptr; + for (const auto& entry : spaces_by_folders) { + if (!entry.first.empty()) { + print_spaces(ctx.out() << '\n', entry.second, ", in folder " + entry.first); + } else { + assert(entry.second.size() == 1); + dflt_space = entry.second[0]; + } + } + + if (dflt_space) { + const auto& space = *dflt_space; + ctx.out() << "\nDefault dataspace (no folder):" << std::endl; + print(space, 0); + } + + const size_t n = spaces.size(); + const size_t m = spaces_by_folders.size() - (dflt_space? 1: 0); + ctx.out() << fmt::format( + "\n{} dataspace{}, in {} folder{}.", + thou(n), (n > 1)? "s": "", thou(m), (m > 1)? "s": "" + ) << std::endl; + } + + if (spaces.size() != total) { + const size_t n = (total - spaces.size()); + ctx.out() << fmt::format( + "\n(Filtered-out {} dataspace{}, out of {})", + thou(n), (n > 1)? "s": "", thou(total) + ) << std::endl; + } + + return true; +} + +void pushResource( + oes::core::JSONWriter& json, + const etp::Datatypes::Object::Resource& res +) { + json.push_object(); + json + .key("uri").string(res.m_uri) + .key("name").string(res.m_name) + .key("lastChanged").number(res.m_lastChanged) + .key("lastChanged-ISO").string( + printMicroSecDateTime(res.m_lastChanged, true, true, false) + ) + ; + + if (res.m_storeLastWrite) { + json + .key("storeLastWrite").number(res.m_storeLastWrite) + .key("lastChanged-ISO").string( + printMicroSecDateTime(res.m_storeLastWrite, true, true, false) + ) + ; + } + + if (!res.m_alternateUris.empty()) { + json + .key("alternateUris") + .value(res.m_alternateUris) + ; + } + + if (res.m_sourceCount) { + json.key("sourceCount").number(*res.m_sourceCount); + } + + if (res.m_targetCount) { + json.key("targetCount").number(*res.m_targetCount); + } + + if (!res.m_customData.empty()) { + json.key("customData") + .push_object(); + for (const auto& entry : res.m_customData) { + etp::dataValueAsJson(json, false, entry.first, entry.second); + } + json.pop_object(); + } + + json.pop_object(); +} + +bool space_toc(const EmlAppContext& ctx) { + const bool utc = ctx.hasArg("utc"); + const bool iso = ctx.hasArg("iso"); + const bool ovr = ctx.hasArg("overwrite"); + const std::string out = ctx.stringArg(ctx.action()); // filename + + std::ofstream fd; + bool do_json = false; + oes::core::JSONWriter json(fd); + + if (!out.empty()) { + if (!tryRemovefile(ctx.err(), out, ovr)) { + return false; + } + + fd.open(out, std::ios::out); + if (!fd.good()) { + ctx.err() << "Error:\tOpening file: " << out << std::endl; + return false; + } + + fd << std::setprecision(std::numeric_limits::digits10 + 1); + do_json = true; + } + + const std::string since = ctx.stringArg("since"); + const int64_t lastWrittenFilter = parseSince(since, utc); + if (lastWrittenFilter < 0) { + ctx.err() << "Error:\tInvalid --since value: " << since << + "\n\tUse an offset: 5min, 1d, 1w; or an ISO-8601 date-time (see --iso)"<< std::endl; + return false; + } else if (ctx.verbose && lastWrittenFilter != 0) { + using namespace std::chrono; + microseconds usec(lastWrittenFilter); // UTC + system_clock::time_point ts(usec); + ctx.msg() << "Resources since " << oes::core::formatIsoDateTime(ts) << " UTC" << std::endl; + } + + const auto [space_path, space_ok] = getSpacePath(ctx); + if (!space_ok) { + return false; + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDiscovery + }); + if (!clt.connect()) { + return false; // error already issued + } + + auto* proto = clt.disco_proto(); + if (!proto) { + return false; + } + + auto space_uri = etp::EmlUri::make_space(space_path); + const std::string types_csv = ctx.stringArg("types"); + std::vector types; + if (!types_csv.empty()) { + boost::split(types, types_csv, boost::algorithm::is_any_of("\t ,;|")); + for (auto& type : types) { + std::vector parts; + boost::split(parts, type, boost::algorithm::is_any_of(".")); + switch (parts.size()) { + case 0: ctx.err() << "Error: Invalid type: " << type << std::endl; return false; + case 1: type = "resqml20." + type; break; + case 2: break; // use as-is + default: ctx.err() << "Error: Invalid type: " << type << std::endl; return false; + } + ctx.msg() << "Filtering on type: " << type << std::endl; + } + } else { + ctx.msg() << "No type-filtering: Getting all types." << std::endl; + } + + auto rpc = rpcFor(proto, &etp::proto12::IDiscovery::GetResources); + rpc.req_.m_context.m_uri = space_uri.to_string(); + rpc.req_.m_context.m_depth = 1; // required by PV's server... See iLab#3 + rpc.req_.m_context.m_dataObjectTypes = types; + if (lastWrittenFilter) { + rpc.req_.m_storeLastWriteFilter = lastWrittenFilter; + } + ctx.out() << "Discovering " << rpc.req_.m_context.m_uri << "..." << std::endl; + + if (do_json) { + json.push_object(); + json + .key("dataspace").string(space_uri.dataspace_) + .key("resources").push_array(); + } + + size_t json_resources = 0; + decltype(rpc.res_.m_resources) all_resources; + + // For now, in case of multi-part response, + // we simply accumulate the multi-part responses, + // instead of having a proper concurrent pipeline. + // TODO: process "chunks" in separate thread, + // feeding each chunk via moodycamel's blocking-RWQueue, + // and using ProcTopN-like collection to discard chunks, + // avoiding large memory use (e.g. 460MB for KernRiver). + if (!rpc.callForEachValue( + ctx.err(), clt.put_get_timeout_, + [&](auto& res) { + if (do_json) { + ++json_resources; + pushResource(json, res); + } else { + all_resources.emplace_back(std::move(res)); + } + } + )) { + return false; + } + + if (do_json) { + json + .pop_array() + .pop_object(); + fd.close(); + } + + auto& rscs = all_resources; + const size_t res_count = std::max<>(rscs.size(), json_resources); + if (res_count == 0) { + if (types.empty() && !lastWrittenFilter) { + ctx.out() << "Empty dataspace." << std::endl; + } else { + ctx.out() << "No resources matching filter(s)." << std::endl; + } + return true; + } + + if (do_json) { + fmt::print( + ctx.out(), "Wrote {} resource{} to JSON file {}\n", + res_count, (res_count == 1)? "": "s", out + ); + return true; + } else { + fmt::print( + ctx.out(), "Got {} resource{}.\n", + res_count, (res_count == 1)? "": "s" + ); + } + + std::sort(rscs.begin(), rscs.end(), [](const auto& lhs, const auto& rhs) { + return lhs.m_uri < rhs.m_uri; + }); + + const size_t topN = std::min<>(rscs.size(), size_t(ctx.args["top"].as())); + + // Gather the range of dates for all resources + size_t lmod_count = 0; + size_t slwr_count = 0; + size_t lmod0_count = 0; + size_t slwr0_count = 0; + int64_t lmod_min = std::numeric_limits::max(); + int64_t lmod_max = 0; + int64_t slwr_min = std::numeric_limits::max(); + int64_t slwr_max = 0; + + for (const auto& rsc : rscs) { + if (0 == rsc.m_lastChanged) { + ++lmod0_count; + } else if (!lmod_count) { + ++lmod_count; + lmod_min = lmod_max = rsc.m_lastChanged; + } else { + ++lmod_count; + lmod_min = std::min<>(lmod_min, rsc.m_lastChanged); + lmod_max = std::max<>(lmod_max, rsc.m_lastChanged); + } + + if (0 == rsc.m_storeLastWrite) { + ++slwr0_count; + } else if (!slwr_count) { + ++slwr_count; + slwr_min = slwr_max = rsc.m_storeLastWrite; + } else { + ++slwr_count; + slwr_min = std::min<>(slwr_min, rsc.m_storeLastWrite); + slwr_max = std::max<>(slwr_max, rsc.m_storeLastWrite); + } + } + + for (size_t i = 0; i < topN; ++i) { + const auto& rsc = rscs[i]; + etp::EmlUri uri = etp::EmlUri::parse(rsc.m_uri); + OES_ASSERT(uri.type_ == etp::EmlUri::eOBJ_INSTANCE, continue;); + std::string lmod = printMicroSecDateTime(rsc.m_lastChanged, utc, iso, ctx.debug); + std::string slwr = printMicroSecDateTime(rsc.m_storeLastWrite, utc, iso, ctx.debug); + fmt::print(ctx.msg(), R"( + URI: {} + Alternate-URIs: {})", + rsc.m_uri, rsc.m_alternateUris.size() + ); + fmt::print(ctx.out(), R"( + Name: {} + Type: {} + Guid: {} + Last-Modified: {} + Store-Last-Write: {} + Custom-Data: {} entr{} +)", + rsc.m_name, uri.to_type(), uri.guid_.toXmlString(), + lmod, slwr, rsc.m_customData.size(), + (rsc.m_customData.size() == 1)? "y": "ies" + ); + } + + ctx.out() << '\n'; + + int width = static_cast(std::log10(rscs.size())); + width += (width / 3); // for commas + width += 1; + const std::string spacer(width + 30, ' '); + + if (topN < rscs.size()) { + fmt::print( + ctx.out(), "({:>{}} not shown. Increase --top to see more)\n", + thou(rscs.size() - topN), width + ); + } + + if (lmod0_count) { + fmt::print( + ctx.out(), "({:>{}} resource{} no Last-Modified timestamp)\n", + thou(lmod0_count), width, (lmod0_count == 1)? " has": "s have" + ); + } + if (slwr0_count) { + fmt::print( + ctx.out(), "({:>{}} resource{} no Store-Last-Write timestamp)\n", + thou(slwr0_count), width, (slwr0_count == 1)? " has": "s have" + ); + } + + if (lmod_min == lmod_max) { + fmt::print( + ctx.out(), "({:>{}} resource{} Last-Modified on {})\n", + thou(lmod_count), width, (lmod_count == 1)? "": "s", + printMicroSecDateTime(lmod_min, utc, iso, ctx.debug) + ); + } else if (lmod_count) { + fmt::print( + ctx.out(), "({:>{}} resource{} Last-Modified between {}\n{}and {})\n", + thou(lmod_count), width, (lmod_count == 1)? "": "s", + printMicroSecDateTime(lmod_min, utc, iso, ctx.debug), spacer, + printMicroSecDateTime(lmod_max, utc, iso, ctx.debug) + ); + } + + if (slwr_min == slwr_max) { + fmt::print( + ctx.out(), "({:>{}} resource{} Store-Last-Write on {})\n", + thou(slwr_count), width, (slwr_count == 1)? "": "s", + printMicroSecDateTime(slwr_min, utc, iso, ctx.debug) + ); + } else if (slwr_count) { + fmt::print( + ctx.out(), "({:>{}} resource{} Store-Last-Write between {}\n{}and {})\n", + thou(slwr_count), width, (slwr_count == 1)? "": "s", + printMicroSecDateTime(slwr_min, utc, iso, ctx.debug), spacer, + printMicroSecDateTime(slwr_max, utc, iso, ctx.debug) + ); + } + + return true; +} + +bool space_stats_via_discovery( + const EmlAppContext& ctx, EtpClient12& clt, + etp::proto12::IDiscovery& proto, const std::string& space_path +) { + auto space_uri = etp::EmlUri::make_space(space_path); + + auto rpc = rpcFor(&proto, &etp::proto12::IDiscovery::GetResources); + + rpc.req_.m_context.m_uri = space_uri.to_string(); + rpc.req_.m_context.m_depth = 1; // required by PV's server... See iLab#3 + rpc.req_.m_context.m_dataObjectTypes.clear(); // i.e. no type filtering + rpc.req_.m_storeLastWriteFilter.reset(); // i.e. no time filtering + ctx.out() << "Discovering " << rpc.req_.m_context.m_uri << "..." << std::endl; + + auto ok = rpc.call(ctx.err(), clt.put_get_timeout_); + if (!ok.first) { + return false; + } + + auto& rscs = rpc.res_.m_resources; + if (rscs.empty()) { + ctx.out() << "No ETP types found" << std::endl; + return true; + } + + std::map types; + for (const auto& rsc : rscs) { + etp::EmlUri uri = etp::EmlUri::parse(rsc.m_uri); + ++types[uri.to_type()]; + } + + for (const auto& entry : types) { + const std::string& name = entry.first; + int32_t count = entry.second; + ctx.out() << fmt::format("{:>18} {}", thou(count), name) << std::endl; + } + ctx.out() << fmt::format("{:>18} {}", thou(rscs.size()), "TOTAL") << std::endl; + return true; +} + +struct AryTypeInfo { + etp::AryType type_; + int elem_size_; + std::string_view label_; +}; + +auto typeOf(etp::AryType type) -> const AryTypeInfo* { + using namespace std::string_view_literals; + static constexpr AryTypeInfo ary_types[] { + { etp::AryType::eArrayOfBoolean, 1, "Boolean"sv }, + { etp::AryType::eArrayOfInt , 4, "Int"sv }, + { etp::AryType::eArrayOfLong , 8, "Long"sv }, + { etp::AryType::eArrayOfFloat , 4, "Float"sv }, + { etp::AryType::eArrayOfDouble , 8, "Double"sv }, + { etp::AryType::eArrayOfString , 0, "String"sv }, // variable size + { etp::AryType::eBytes , 1, "Byte"sv }, + }; + // Do linear search, in case the enum evolves behind our back + for (const AryTypeInfo& ary_type : ary_types) { + if (type == ary_type.type_) { + return &ary_type; + } + } + return nullptr; +} + +bool space_stats(const EmlAppContext& ctx) { + const auto [space_path, space_ok] = getSpacePath(ctx); + if (!space_ok) { + return false; + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDiscovery, + etp::data12::Protocol::eSupportedTypes + }); + if (!clt.connect()) { + return false; // error already issued + } + + // If SupportedTypes proto not supported, do it via Discovery instead. + // Can be tested via `--shun-protocols types` for example. + auto* proto = clt.types_proto(); + if (!proto) { + auto* proto2 = clt.disco_proto(); + if (proto2) { + ctx.out() << "Warning: No SupportedTypes(25) support: " + "Using slower Discovery(3) instead" << std::endl; + return space_stats_via_discovery(ctx, clt, *proto2, space_path); + } + ctx.err() << "Error: Server supports neither SupportedTypes(25) nor Discovery(3) protocols" << std::endl; + return false; + } + + auto uri = etp::EmlUri::make_space(space_path); + + auto rpc = rpcFor(proto, &etp::proto12::ISupportedTypes::GetSupportedTypes); + + rpc.req_.m_uri = uri.to_string(); // init request + rpc.req_.m_scope = etp::data12::Object::ContextScopeKind::eSelf; // ignored + rpc.req_.m_countObjects = true; + rpc.req_.m_returnEmptyTypes = false; + + auto ok = rpc.call(ctx.err(), clt.put_get_timeout_); + if (!ok.first) { + return false; + } + + auto& types = rpc.res_.m_supportedTypes; + if (types.empty()) { + ctx.out() << "No ETP types found" << std::endl; + return true; + } + + // sort, for deterministic order + std::sort(types.begin(), types.end(), [](const auto& lhs, const auto& rhs) { + return lhs.m_dataObjectType < rhs.m_dataObjectType; + }); + + int64_t total = 0; + for (const auto& entry : types) { + const std::string& name = entry.m_dataObjectType; + int32_t count = !entry.m_objectCount? 0: *entry.m_objectCount; + if (count > 0) { + total += count; + } + ctx.out() << fmt::format("{:>18} {}", thou(count), name) << std::endl; + } + ctx.out() << fmt::format("{:>18} {}", thou(total), "TOTAL") << std::endl; + return true; +} + +bool space_array_stats(const EmlAppContext& ctx) { + const auto [space_path, space_ok] = getSpacePath(ctx); + if (!space_ok) { + return false; + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDataArray + }); + if (!clt.connect()) { + return false; // error already issued + } + + auto* proto = clt.array_proto(); + if (!proto) { + return false; + } + + auto rpc = rpcFor(proto, &etp::proto12::IDataArray::GetDataArrayMetadata); + rpc.req_ = etp::DataArrayExt::to_list_arrays_req(space_path); + + auto ok = rpc.call(ctx.err(), clt.put_get_timeout_); + if (!ok.first) { + return false; + } + + auto arrays = etp::DataArrayExt::move_to_info_vec(rpc.res_.m_arrayMetadata); + if (arrays.empty()) { + ctx.out() << "No ETP arrays found" << std::endl; + return true; + } + + const size_t topN = std::max<>(1, ctx.args["top"].as()); // 10 by default + std::multimap topMap; + + std::vector bytes_histogram_; + bytes_histogram_.reserve(32); + + size_t errors = 0; + std::vector countByRank(8); + std::map countByType; + std::map countByObjType; + std::map bytesByObjType; + for (const auto& entry : arrays) { + etp::EmlUri uri = etp::EmlUri::parse(entry.ident_.m_uri); + if (uri.type_ != etp::EmlUri::eOBJ_INSTANCE) { + ++errors; + continue; + } + const size_t rank = entry.meta_.m_dimensions.size(); + if (!(0 < rank && rank < countByRank.size())) { + ++errors; + continue; + } + + etp::Datatypes::AnyArrayType type = entry.meta_.m_transportArrayType; + if (!(etp::AryType::eArrayOfBoolean <= type && type <= etp::AryType::eBytes)) { + ++errors; + continue; + } + + const AryTypeInfo* tinfo = typeOf(type); + const int elem_bytes = tinfo? tinfo->elem_size_ : -1; + if (elem_bytes < 0) { + ++errors; + continue; + } + size_t bytes = elem_bytes; + if (bytes >= 1) { + for (int64_t dim : entry.meta_.m_dimensions) { + bytes *= dim; + } + } + + ++countByRank[rank]; + ++countByType[type]; + ++countByObjType[uri.obj_type_]; + + if (bytes == 0) { + // e.g. 2D point array with 0x3 dimensions. + // Not sure that's normal though... + if (bytes_histogram_.empty()) { + bytes_histogram_.emplace_back(1); + } else { + ++bytes_histogram_[0]; + } + continue; + } + + bytesByObjType[uri.obj_type_] += bytes; + + if (topMap.size() < topN) { + topMap.emplace(bytes, &entry); + } else if (topMap.begin()->first < bytes) { + topMap.erase(topMap.begin()); + topMap.emplace(bytes, &entry); + } + + const size_t bucket = 1 + static_cast(std::log2(bytes)); + if (bucket >= bytes_histogram_.size()) { + bytes_histogram_.resize(bucket + 1); + } + ++bytes_histogram_[bucket]; + } + + auto& os = ctx.out(); + + os << fmt::format("{:>18} by rank >>>", "<<< Arrays") << std::endl; + for (size_t rank = 0; rank < countByRank.size(); ++rank) { + if (size_t n = countByRank[rank]) { + os << fmt::format("{:>18} {}D", thou(n), rank) << std::endl; + } + } + + os << fmt::format("\n{:>18} by element type >>>", "<<< Arrays") << std::endl; + for (const auto& entry : countByType) { + if (const AryTypeInfo* tinfo = typeOf(entry.first)) { + if (size_t n = entry.second) { + os << fmt::format("{:>18} {}", thou(n), tinfo->label_) << std::endl; + } + } + } + + os << fmt::format("\n{:>18} by bytes-range >>>", "<<< Arrays") << std::endl; + if (!bytes_histogram_.empty() && bytes_histogram_[0]) { + os << fmt::format("{:>18} empty", thou(bytes_histogram_[0])) << std::endl; + } + for (size_t i = 1; i < bytes_histogram_.size(); ++i) { + const size_t width = thou(size_t(pow(2, bytes_histogram_.size() - 1))).size(); + const size_t count = bytes_histogram_[i]; + std::ostream& os2 = count? os: ctx.msg(); + os2 << fmt::format( // show empty range in verbose mode only + "{:>18} ({:>{}} - {:>{}}] bytes", thou(count), + thou(size_t(pow(2, i - 1))), width, + thou(size_t(pow(2, i))), width + ) << std::endl; + } + + int width = static_cast(std::log10(arrays.size())); + width += (width / 3); // for commas + width += 1; + + size_t total_bytes = 0; + os << fmt::format("\n{:>18} by object type >>>", "<<< Arrays") << std::endl; + for (const auto& count_entry : countByObjType) { + const std::string& obj_type = count_entry.first; + const size_t count = count_entry.second; + const size_t bytes = bytesByObjType[obj_type]; + os << fmt::format("{:>18} {:>{}}x {}", thou(bytes), thou(count), width, obj_type) << std::endl; + total_bytes += bytes; + } + os << fmt::format("{:>18} {:>{}}x TOTAL", thou(total_bytes), thou(arrays.size()), width) << std::endl; + + os << fmt::format("\n{:>18} bytes - Top {} >>>", "<<< Arrays", topN) << std::endl; + for (const auto& entry : oes::core::reverse_of(topMap)) { + const size_t bytes = entry.first; + const etp::AryInfo& ary = *entry.second; + etp::EmlUri uri = etp::EmlUri::parse(ary.ident_.m_uri); + const AryTypeInfo* tinfo = typeOf(ary.meta_.m_transportArrayType); + const auto& elem_type = tinfo? tinfo->label_: "???"; + const size_t rank = ary.meta_.m_dimensions.size(); + std::string dims = thou(ary.meta_.m_dimensions[0]); + for (size_t i = 1; i < rank; ++i) { + dims += " x "; + dims += thou(ary.meta_.m_dimensions[i]); + } + + os << fmt::format( + "{:>18} {}({})\n{:>18} {}\n{:>18} {}D - {} - {}", + thou(bytes), uri.obj_type_, uri.guid_.toXmlString(), + "", ary.ident_.m_pathInResource, + "", rank, dims, elem_type + ) << std::endl; + } + + return true; +} + +bool space_delete(const EmlAppContext& ctx) { + const auto [space_path, space_ok] = getSpacePath(ctx, true, false); + if (!space_ok) { + return false; + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDataspace + }); + if (!clt.connect()) { + return false; + } + + auto* proto = clt.space_proto(); + if (!proto) { + return false; + } + + // DeleteDS needs uids, while users/humans know about paths. + // So must look for dataspaces to find the uids. + auto get_rpc = rpcFor(proto, &etp::proto12::IDataspace::GetDataspaces); + auto del_rpc = rpcFor(proto, &etp::proto12::IDataspace::DeleteDataspaces); + + auto get_ok = get_rpc.call(ctx.err(), clt.put_get_timeout_); + if (!get_ok.first) { + return false; + } + + const auto& spaces = get_rpc.res_.m_dataspaces; + for (const auto& space : spaces) { + if (space.m_path == space_path) { + del_rpc.req_.m_uris.emplace(space_path, space.m_uri); + break; + } + } + + if (del_rpc.req_.m_uris.empty()) { + ctx.err() << "Error: ETP Dataspace not found: " << space_path << std::endl; + return false; + } + + auto del_ok = del_rpc.call(ctx.err(), clt.put_get_timeout_); + if (!del_ok.first) { + return false; + } + + return true; +} + +bool space_delete_all(const EmlAppContext& ctx) { + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eDataspace + }); + if (!clt.connect()) { + return false; + } + + size_t total = 0; + auto spaces = clt.globSpaces(ctx.stringArg(ctx.action()), &total); + if (spaces.empty()) { + ctx.out() << "Nothing to delete" << std::endl; + return true; + } + + auto* proto = clt.space_proto(); + assert(proto); // globSpaces() throws on no DS proto + + auto del_rpc = rpcFor(proto, &etp::proto12::IDataspace::DeleteDataspaces); + for (const auto& space : spaces) { + del_rpc.req_.m_uris.emplace(space.m_uri, space.m_uri); + } + + if (del_rpc.req_.m_uris.empty()) { + ctx.out() << "Nothing to delete" << std::endl; + return true; + } + + auto del_ok = del_rpc.call(ctx.err(), clt.put_get_timeout_); + if (!del_ok.first) { + return false; + } + + const size_t n = spaces.size(); + ctx.out() << fmt::format( + "Deleted {} ETP dataspace{}, out of {}", + thou(n), (n > 1)? "s": "", thou(total) + ) << std::endl; + return true; +} + +//============================================================================ + +bool space_import_epc(const EmlAppContext& ctx) { + // Note: Prolog mostly the same as in openkv_import_epc() + const std::string epc = ctx.stringArg(ctx.action()); + if (epc.empty()) { + ctx.err() << "Error: Missing required argument to --" << ctx.action() << std::endl; + return false; + } + + fs::path epc_path(epc); + fs::file_status epc_stat = fs::status(epc_path); + if (!fs::is_regular_file(epc_stat)) { + ctx.err() << "Error: EPC file not found: " << epc << std::endl; + return false; + } + + fs::path hdf_path(epc); + hdf_path.replace_extension(".h5"); + const std::string hdf = hdf_path.string(); + fs::file_status hdf_stat = fs::status(hdf_path); + if (!fs::is_regular_file(hdf_stat)) { + ctx.err() << "Error: HDF5 file not found: " << hdf << std::endl; + return false; + } + + // space-path should be of the form folder/name, where + // folder shouldn't contain any back-slashes, and at most a single slash, + // If name is omitted, it comes from the EPC filename. + // If --space-path completely omitted, uses default dataspace. + auto [space_path, space_ok] = getSpacePath(ctx); + if (!space_ok) { + return false; + } + + // Probably close to what ProtocolCapabilityKind::eMaxDataItemCount should be + // (max_batch_size == 0) disabled by-count batching (remains by-bytes batching) + int max_batch_size = ctx.args["max-batch-size"].as(); + if (!(0 <= max_batch_size && max_batch_size < 10*1000)) { + ctx.err() << "Error: Invalid --max-batch-size: Not in [0 - 10,000] range" << std::endl; + return false; + } + + // space_path may be just a folder, or a folder/name. + // block below possibly edits space_path, from EPC's name. + if (!space_path.empty()) { + std::string path, err_msg; + std::string folder, name, db_path; + const bool path_ok = oes::eml::openkv::Space::validatePath( + space_path, epc_path.string(), + path, db_path, folder, name, err_msg // outputs + ); + if (!path_ok) { + ctx.err() << "Error: Invalid --space-path value: " << space_path + << "\n\t" << err_msg << std::endl; + return false; + } + space_path.swap(path); + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eStore, + etp::data12::Protocol::eDataArray, + etp::data12::Protocol::eTransaction + }); + if (!clt.connect()) { + return false; + } + + return etp::importEpc( + *clt.clt_, space_path, epc, hdf, + clt.put_get_timeout_, size_t(max_batch_size) + ); +} + +//============================================================================ + +bool space_export_epc(const EmlAppContext& ctx) { + const auto [space_path, space_ok] = getSpacePath(ctx); + if (!space_ok) { + return false; + } + + std::string epc = ctx.stringArg(ctx.action()); + const bool epc_was_empty = epc.empty(); + if (epc_was_empty) { + // Infer the filename from the space_path + etp::EmlUri uri = etp::EmlUri::make_space(space_path); + std::string folder = uri.dataspaceFolder(); + if (folder.empty()) { + epc = fmt::format("{}.epc", scrubFilename(uri.dataspace_)); + } else { + epc = fmt::format("{}@{}.epc", scrubFilename(uri.dataspaceName()), scrubFilename(folder)); + } + } + fs::path epc_path(epc); + if (!epc_path.has_extension()) { + epc_path.replace_extension(".epc"); + } + if (epc_was_empty) { + epc_path = epc_path.filename(); // strip leading folder + } + epc = epc_path.string(); + + // check file doesn't exist or can be overwritten, before too much I/O + const bool overwrite = ctx.hasArg("overwrite"); + if (!tryRemovefile(ctx.err(), epc, overwrite)) { + return false; + } + + // Check the folder is writable too, in case user opened DOS Console + // in UAC-protected C:\Program Files subfolder, and forgot to CD elsewhere. + // Both HDF5 and ZIP fail horribly in that case, with tons of errors. + const bool can_write = canWriteFileInFolder(epc); + if (!can_write) { + namespace fs = fs; + fs::path etp_path(epc); + fs::path dest_dir(absolute(etp_path).parent_path()); + ctx.err() << "Error: Cannot write to folder " << dest_dir.string() << std::endl; + if (!ctx.hasArg("force")) { // allow bypassing that check, in case it's buggy... + return false; // bail out + } + } + + fs::path hdf_path(epc_path); + hdf_path.replace_extension(".h5"); + const std::string hdf = hdf_path.string(); + + // Probably close to what ProtocolCapabilityKind::eMaxDataItemCount should be + // (max_batch_size == 0) disabled by-count batching (remains by-bytes batching) + const int max_batch_size = ctx.args["max-batch-size"].as(); + if (!(0 <= max_batch_size && max_batch_size <= 10*1000)) { + ctx.err() << "Error: Invalid --max-batch-size: Not in [0 - 10,000] range" << std::endl; + return false; + } + + // Peak RAM will be proportional to (constant + (max_queue_depth * max_msg_size)), + // which by default would be (constant + (10 * 1MB)). Given that the producer side + // of the queue (DataArray.Get messages) are much slower than the consumer side + // (HDF5 write), the queue is in fact unlikely to get full, and the consumer side + // will be "stalled" most of the time, waiting for new entries to be enqueued. + const int max_queue_depth = ctx.args["array-queue-depth"].as(); + if (!(0 <= max_batch_size && max_batch_size <= 1000)) { + ctx.err() << "Error: Invalid --array-queue-depth: Not in [0 - 1,000] range" << std::endl; + return false; + } + + EtpClient12 clt(ctx); + clt.cfg_.requested_protocols_.assign({ + etp::data12::Protocol::eTransaction, + etp::data12::Protocol::eDiscovery, + etp::data12::Protocol::eStore, + etp::data12::Protocol::eDataArray + }); + if (!clt.connect()) { + return false; // error already issued + } + + etp::ExportEpcOptions options; + + options.timeout_ = clt.put_get_timeout_; + options.max_batch_size_ = size_t(max_batch_size); + options.overwrite_ = overwrite; + + options.max_queue_depth_ = max_queue_depth; + options.hdf5_compression_ = hdf5CompressionLevel(ctx); + options.hdf5_large_data_compression_ = hdf5LargeDataCompressionLevel(ctx); + + options.out_ = ctx.quiet? nullptr: &ctx.out(); + options.msg_ = ctx.verbose? &ctx.msg(): nullptr; + options.warn_ = ctx.quiet? nullptr: &ctx.out(); + options.err_ = ctx.quiet? nullptr: &ctx.err(); + + const bool multi_threaded = ctx.hasArg("jobs"); + if (!multi_threaded) { + ctx.dbg() << "Using single-threaded, load-all-then-write EPC export." << std::endl; + return exportEpc(*clt.clt_, space_path, epc, hdf, options); + } + + // single-threaded when -j is not explicitly specified + int64_t jobs = ctx.args["jobs"].as(); + const auto max_jobs = std::thread::hardware_concurrency(); + if (jobs && !(3 <= jobs && jobs <= max_jobs)) { + fmt::print(ctx.err(), "Error: Invalid --jobs: Not in [3 - {}] range\n", max_jobs); + return false; + } + options.thread_count_ = static_cast(jobs); + ctx.dbg() << "Using multi-Threaded, pipelined load-one-and-write EPC export." << std::endl; + return exportEpcMt(*clt.clt_, space_path, epc, hdf, options); +} + +//============================================================================ + +} //namespace + +namespace oes::apps::eml { + +/****************************************************************************/ + +int space_main(EmlAppContext& ctx, const UnparsedArgs& unparsed_args) { + po::options_description cmd_desc( + "\nUsage: openETPServer [global-options] " + ctx.command + " --action [args]\n" + "\nActions", ctx.terminalWidth_ + ); + cmd_desc.add_options() + ("help,h", "Display this information") + + ("list,l", txtArgOpt(), "List ETP dataspaces (aka Projects)") + ("info", "Show info / details on ETP dataspace") + ("toc", txtArgOpt(), "Get Table-of-Contents on ETP dataspace\n" + "Prints Top-N (no argument); or writes JSON file") + ("stats", "Get Statistics on ETP dataspace") + ("array-stats", "Get Statistics on ETP dataspace bulk data arrays") + ("new", "Create new ETP dataspace") + ("delete", "Delete ETP dataspace") + ("delete-all", txtArg(), "Delete All matching ETP dataspaces") + + ("import-epc", txtArg(), "Import (PUT) EPC file into ETP server") + ("export-epc", txtArgOpt(), "Export (GET) EPC file from ETP server") + ; + + po::options_description opt_desc("Options", ctx.terminalWidth_); + opt_desc.add_options() + ("space-path,s", txtArg(), "ETP dataspace path (folder/project)") + ("space-guid", txtArgOpt(), "ETP dataspace guid (uses path if omitted)") + ("top", intArg(10), "How many entries to list") + ("utc", "Show timestamps in UTC, instead of local time") + ("iso", "Show timestamps in ISO-8601 format") + ("flat", "Do not group ETP dataspaces by folder") + ("since", txtArg(), "Filter by Last-Store-Write timestamp.\n" + "(ISO date-time; or duration: 1d, 3m, 1y, etc...)") + ("types", txtArg(), "Filter TOC by object types (CSV).\n") + ("xdata-file", txtArg(), "New Space meta-data as JSON file") + ("overwrite", "Overwrite existing EPC on export") + ("force", "Continue instead of exit, on some warnings/errors") + + ("jobs,j", + greedy_value()->implicit_value(0), + "How many worker-threads:\n" + "* not specified: Defaults to single-threaded;\n" + "* -j: Use as many threads as hardware cores;\n" + "* -jN: Use exactly N threads") + + ("array-queue-depth", intArg(10), "Max (sub)arrays accumulated in-memory\n" + "(Peak RAM proportional to -M x this-option)") + + ("hdf5-compression,z", intArg(3), "Compression level of most datasets:\n" + "0 none; 1 fast but bigger; 9 best but slower") + ("hdf5-large-data-compression", intArg(3), "Compression level of large datasets (3D or more):\n" + "0 none; 1 fast but bigger; 9 best but slower") + ; + + cmd_desc.add(opt_desc); + cmd_desc.add(connect_options(ctx.terminalWidth_)); + cmd_desc.add(authenticate_options(ctx.terminalWidth_)); + + if (int rc = ctx.reparse(cmd_desc, unparsed_args)) { + return (rc == -1)? 0: rc; + } + + EmlAppNamedAction actions[] { + { "new", &space_new }, + { "info", &space_info }, + { "list", &space_list }, + { "toc", &space_toc }, + { "stats", &space_stats }, + { "array-stats", &space_array_stats }, + { "delete", &space_delete }, + { "delete-all", &space_delete_all }, + { "import-epc", &space_import_epc }, + { "export-epc", &space_export_epc }, + }; + return ctx.dispatchActions(actions); +} + +/****************************************************************************/ + +} // namespace oes::apps::eml diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/files.cmake b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/files.cmake new file mode 100644 index 0000000000000000000000000000000000000000..4715c8f717800f4d5a7d50714710996776ca2435 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/files.cmake @@ -0,0 +1,19 @@ +# List all files required for openETPServer executable + +################################################################################ +# root directory +set( openETPServer_SOURCES + main.cpp + EmlAppContext.cpp + EtpClient12.cpp + Hdf5Cmds.cpp + ProbeCmds.cpp + ServerCmds.cpp + SpaceCmds.cpp +) +SOURCE_GROUP( "" FILES ${openETPServer_SOURCES} ) +set( openETPServer_HEADERS + EmlAppContext.h + EtpClient12.h +) +SOURCE_GROUP( "" FILES ${openETPServer_HEADERS} ) diff --git a/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/main.cpp b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4191fca0995ada765bdb0f37fd6d393cabcfa007 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/bin/openETPServer/main.cpp @@ -0,0 +1,230 @@ +// ============================================================================ +// Copyright 2015-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "EmlAppContext.h" + +#include +#include +#include + +#include // for nullptr +#include +#include +#include +#include + +#include +#include +#include + +namespace po = boost::program_options; +using namespace oes::apps::eml; + +namespace { + +std::weak_ptr global_ctx__; + +void handlePscAssertion(const oes::core::AssertionInfo& info) { + std::shared_ptr ctx_ptr(global_ctx__.lock()); + // ctx_ptr->err() obeys the --quiet flag, if present. + std::ostream& err = ctx_ptr? ctx_ptr->err(): std::cerr; + err << info.file << '(' << info.line << "): " << info.message + << ": " << info.condition +// << " in " << info.function + << std::endl; +} + +class OutputFilter : public oes::core::MessageFilter { +public: + virtual bool filter(oes::core::Message_ptr message) const { + std::shared_ptr ctx_ptr(global_ctx__.lock()); + if (ctx_ptr && ctx_ptr->quiet) { + return true; + } + if (ctx_ptr) { + using oes::core::Message; + Message::Severity msg_sev = message->severity(); + std::ostream& ostr = + msg_sev == Message::status ? ctx_ptr->msg() : + msg_sev == Message::notice ? ctx_ptr->msg() : + msg_sev <= Message::debug ? ctx_ptr->dbg() : // unknown, status, debug + msg_sev < Message::warning ? ctx_ptr->msg() : // info + msg_sev == Message::warning ? ctx_ptr->out() : // warning + msg_sev >= Message::error ? ctx_ptr->err() : // error, alert, critical, fatal + ctx_ptr->err(); + ostr << message->prefix() << ": " << message->format(); + return true; + } + return false; + } +}; + +} // namespace + +/*! + * \brief Command-Line program for Data-Exchange Resqml. + * + * \param argc the number of command line arguments. + * \param argv the command line arguments, including the executed program name/path. + * \return 0 on success; non-zero otherwise. + */ +int main(int argc, char* argv[]) { +#ifdef OES_DEBUG_BUILD + // For early startup logging, before ctx is created. + for (int i = 0; i < argc; ++i) { + using namespace std; + if (0 == strcmp("--debug-main", argv[i])) { + cout << "Entered main(), static initializations done." << endl; + const int w = static_cast(log10(argc)) + 1; + for (int j = 0; j < argc; ++j) { + cout << fmt::format("ARG#{:0{}} <<{}>>", j, w, argv[j]) << endl; + } + break; + } + } +#endif + + oes::core::cachedTzOffset(); + + const auto tw = EmlAppContext::terminalWidth(); + po::options_description generic_desc("" /*"Global Options"*/, tw); + generic_desc.add_options() +("help,h", "Display this information") +("version,V", "Display version information") +("verbose,v", "Output information messages") +("debug", "Output debug messages") +("quiet,q", "Do not output any messages") +("timer", "Display real/user time of command action") +("output-file,o", txtArg(), "File to write dumps/text to") +#ifdef OES_DEBUG_BUILD +("nobps", "No breakpoints") +#endif + ; + + options_description hidden_desc("Hidden Options", tw); + hidden_desc.add_options() +("command", txtArg(), "Command to execute") +("cmdargs", po::value(), "Arguments for command") +#ifdef OES_DEBUG_BUILD +("debug-main", "Low-level main() logging for startup debugging") +#endif +; + + po::options_description all_desc("All Options", tw); + all_desc.add(generic_desc).add(hidden_desc); + + const EmlAppNamedCommandVec commands = { + { "hdf5", &hdf5_main, "Tools to analyze HDF5 files" }, + { "server", &server_main, "ETP 1.2 server" }, + { "probe", &probe_main, "HTTP query and ETP ping ETP servers" }, + { "space", &space_main, "ETP 1.2 client accessing ETP Dataspaces" } + }; + + std::ostringstream oss; + for (const EmlAppNamedCommand& entry : commands) { + oss << fmt::format(" {:<8} {}\n", entry.name, entry.desc); + } + oss << "\nGlobal Options"; // trick for prettier output. + const std::string& command_list = oss.str(); + + po::options_description visible_desc( + "\nUsage: openETPServer [global-options] --action [options] [args]\n\n" + "Type 'openETPServer -h' for help on a specific command.\n\n" + "Available Commands:\n" + command_list + ); + visible_desc.add(generic_desc); + + // from http://stackoverflow.com/questions/15541498 + // see also https://gist.github.com/randomphrase/10801888 + po::positional_options_description pos; + pos .add("command", 1) + .add("cmdargs", -1); + + variables_map global_args; + try { + parsed_options parsed = po::command_line_parser(argc, argv) + .options(all_desc) + .positional(pos) + .allow_unregistered() + .run(); + + po::store(parsed, global_args); + + if (global_args.size() == 0) { + std::cout << visible_desc << std::endl; + return 0; + } + + std::shared_ptr ctx_ptr(new EmlAppContext(global_args)); + const EmlAppContext& ctx(*ctx_ptr); + global_ctx__ = ctx_ptr; + + if (global_args.count("help") && ctx.command.empty()) { + // global help + ctx.out() << visible_desc << std::endl; + return 0; + } + + if (global_args.count("version")) { + ctx.out() << +R"(Energistics Markup Languages (EML) Command Line Utility. v1 Beta.)" + << std::endl; + // TODO: show OCI version, HDF5 version, etc... + return 0; + } + + if (ctx.command.empty()) { + ctx.err() << "openETPServer: Error: Missing command" << std::endl; + return 1; + } + +#ifdef OES_DEBUG_BUILD + if (ctx.hasArg("nobps")) { + oes::core::setBreakpointHandler([]() { + return false; // should not break + }); + } +#endif + + oes::core::setAssertionHandler(handlePscAssertion); + oes::core::MessageFilter_ptr log_filter(new OutputFilter); + oes::core::Logger::instance().connectFilter(log_filter); + + const int rc = ctx_ptr->dispatchCommand(parsed, commands); + + if (rc == -1) { + ctx.err() << "openETPServer: Error: Unknown command: " << ctx.command + << '\n' << '\n' << visible_desc << std::endl; + } + +#ifdef OES_DEBUG_BUILD + else if (rc != 0) { + std::cerr << "openETPServer: Error: Non-zero return code: " << rc << std::endl; + } else { + std::cout << "openETPServer: done." << std::endl; + } +#endif + + return rc; + } catch (const std::exception& ex) { + std::cerr << "openETPServer: Error: " << ex.what() + << '\n' << '\n' << visible_desc << std::endl; + return 3; // keep in sync with tryCall + } catch (...) { + std::cerr << "openETPServer: Unknown Error" << std::endl; + return 4; // keep in sync with tryCall + } +} diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/lib/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..edb3348c9197ed4c212f29e5902ad2d78a9e9361 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/CMakeLists.txt @@ -0,0 +1,2 @@ + +add_subdirectory(oes) diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/README.md b/Reservoir DMS Suite/open-etp-server/src/lib/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1da5e79ddc182c76eca2bfbdea8d4026db0057ef --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/README.md @@ -0,0 +1,10 @@ +## Reader-Writer-Queue +From: https://github.com/cameron314/readerwriterqueue/releases/tag/v1.0.5 +commit 09732844aff873e8ec489e9005c775b6b1c58fa2 +Note: Modified to not conflict with Qt macro (will be part of next release) + +Files and Folders: +- `moodycamel/spsc/` + +Note: `spsc` stands for Single Producer Single Consumer + diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/LICENSE.md b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..7b667d9e66c56cdf3afbe1d3afafdb81a545352f --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/LICENSE.md @@ -0,0 +1,28 @@ +This license applies to all the code in this repository except that written by third +parties, namely the files in benchmarks/ext, which have their own licenses, and Jeff +Preshing's semaphore implementation (used in the blocking queues) which has a zlib +license (embedded in atomicops.h). + +Simplified BSD License: + +Copyright (c) 2013-2021, Cameron Desrochers +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of +conditions and the following disclaimer in the documentation and/or other materials +provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/README.md b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3c9eec57028c20a6d55e95466ef305a38fee43a9 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/README.md @@ -0,0 +1,155 @@ +# A single-producer, single-consumer lock-free queue for C++ + +This mini-repository has my very own implementation of a lock-free queue (that I designed from scratch) for C++. + +It only supports a two-thread use case (one consuming, and one producing). The threads can't switch roles, though +you could use this queue completely from a single thread if you wish (but that would sort of defeat the purpose!). + +Note: If you need a general-purpose multi-producer, multi-consumer lock free queue, I have [one of those too][mpmc]. + +This repository also includes a [circular-buffer SPSC queue][circular] which supports blocking on enqueue as well as dequeue. + + +## Features + +- [Blazing fast][benchmarks] +- Compatible with C++11 (supports moving objects instead of making copies) +- Fully generic (templated container of any type) -- just like `std::queue`, you never need to allocate memory for elements yourself + (which saves you the hassle of writing a lock-free memory manager to hold the elements you're queueing) +- Allocates memory up front, in contiguous blocks +- Provides a `try_enqueue` method which is guaranteed never to allocate memory (the queue starts with an initial capacity) +- Also provides an `enqueue` method which can dynamically grow the size of the queue as needed +- Also provides `try_emplace`/`emplace` convenience methods +- Has a blocking version with `wait_dequeue` +- Completely "wait-free" (no compare-and-swap loop). Enqueue and dequeue are always O(1) (not counting memory allocation) +- On x86, the memory barriers compile down to no-ops, meaning enqueue and dequeue are just a simple series of loads and stores (and branches) + + +## Use + +Simply drop the readerwriterqueue.h (or readerwritercircularbuffer.h) and atomicops.h files into your source code and include them :-) +A modern compiler is required (MSVC2010+, GCC 4.7+, ICC 13+, or any C++11 compliant compiler should work). + +Note: If you're using GCC, you really do need GCC 4.7 or above -- [4.6 has a bug][gcc46bug] that prevents the atomic fence primitives +from working correctly. + +Example: + +```cpp +using namespace moodycamel; + +ReaderWriterQueue q(100); // Reserve space for at least 100 elements up front + +q.enqueue(17); // Will allocate memory if the queue is full +bool succeeded = q.try_enqueue(18); // Will only succeed if the queue has an empty slot (never allocates) +assert(succeeded); + +int number; +succeeded = q.try_dequeue(number); // Returns false if the queue was empty + +assert(succeeded && number == 17); + +// You can also peek at the front item of the queue (consumer only) +int* front = q.peek(); +assert(*front == 18); +succeeded = q.try_dequeue(number); +assert(succeeded && number == 18); +front = q.peek(); +assert(front == nullptr); // Returns nullptr if the queue was empty +``` + +The blocking version has the exact same API, with the addition of `wait_dequeue` and +`wait_dequeue_timed` methods: + +```cpp +BlockingReaderWriterQueue q; + +std::thread reader([&]() { + int item; +#if 1 + for (int i = 0; i != 100; ++i) { + // Fully-blocking: + q.wait_dequeue(item); + } +#else + for (int i = 0; i != 100; ) { + // Blocking with timeout + if (q.wait_dequeue_timed(item, std::chrono::milliseconds(5))) + ++i; + } +#endif +}); +std::thread writer([&]() { + for (int i = 0; i != 100; ++i) { + q.enqueue(i); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +}); +writer.join(); +reader.join(); + +assert(q.size_approx() == 0); +``` + +Note that `wait_dequeue` will block indefinitely while the queue is empty; this +means care must be taken to only call `wait_dequeue` if you're sure another element +will come along eventually, or if the queue has a static lifetime. This is because +destroying the queue while a thread is waiting on it will invoke undefined behavior. + +The blocking circular buffer has a fixed number of slots, but is otherwise quite similar to +use: + +```cpp +BlockingReaderWriterCircularBuffer q(1024); // pass initial capacity + +q.try_enqueue(1); +int number; +q.try_dequeue(number); +assert(number == 1); + +q.wait_enqueue(123); +q.wait_dequeue(number); +assert(number == 123); + +q.wait_dequeue_timed(number, std::chrono::milliseconds(10)); +``` + + +## CMake installation +As an alternative to including the source files in your project directly, +you can use CMake to install the library in your system's include directory: + +``` +mkdir build +cd build +cmake .. +make install +``` + +Then, you can include it from your source code: +``` +#include +``` + +## Disclaimers + +The queue should only be used on platforms where aligned integer and pointer access is atomic; fortunately, that +includes all modern processors (e.g. x86/x86-64, ARM, and PowerPC). *Not* for use with a DEC Alpha processor (which has very weak memory ordering) :-) + +Note that it's only been tested on x86(-64); if someone has access to other processors I'd love to run some tests on +anything that's not x86-based. + +## More info + +See the [LICENSE.md][license] file for the license (simplified BSD). + +My [blog post][blog] introduces the context that led to this code, and may be of interest if you're curious +about lock-free programming. + + +[blog]: http://moodycamel.com/blog/2013/a-fast-lock-free-queue-for-c++ +[license]: LICENSE.md +[benchmarks]: http://moodycamel.com/blog/2013/a-fast-lock-free-queue-for-c++#benchmarks +[gcc46bug]: http://stackoverflow.com/questions/16429669/stdatomic-thread-fence-has-undefined-reference +[mpmc]: https://github.com/cameron314/concurrentqueue +[circular]: readerwritercircularbuffer.h diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/atomicops.h b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/atomicops.h new file mode 100644 index 0000000000000000000000000000000000000000..b8ff285d529e7538f506e18a96658ee6a63ff600 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/atomicops.h @@ -0,0 +1,679 @@ +// ©2013-2016 Cameron Desrochers. +// Distributed under the simplified BSD license (see the license file that +// should have come with this header). +// Uses Jeff Preshing's semaphore implementation (under the terms of its +// separate zlib license, embedded below). + +#pragma once + +// Provides portable (VC++2010+, Intel ICC 13, GCC 4.7+, and anything C++11 compliant) implementation +// of low-level memory barriers, plus a few semi-portable utility macros (for inlining and alignment). +// Also has a basic atomic type (limited to hardware-supported atomics with no memory ordering guarantees). +// Uses the AE_* prefix for macros (historical reasons), and the "moodycamel" namespace for symbols. + +#include +#include +#include +#include +#include +#include + +// Platform detection +#if defined(__INTEL_COMPILER) +#define AE_ICC +#elif defined(_MSC_VER) +#define AE_VCPP +#elif defined(__GNUC__) +#define AE_GCC +#endif + +#if defined(_M_IA64) || defined(__ia64__) +#define AE_ARCH_IA64 +#elif defined(_WIN64) || defined(__amd64__) || defined(_M_X64) || defined(__x86_64__) +#define AE_ARCH_X64 +#elif defined(_M_IX86) || defined(__i386__) +#define AE_ARCH_X86 +#elif defined(_M_PPC) || defined(__powerpc__) +#define AE_ARCH_PPC +#else +#define AE_ARCH_UNKNOWN +#endif + + +// AE_UNUSED +#define AE_UNUSED(x) ((void)x) + +// AE_NO_TSAN +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define AE_NO_TSAN __attribute__((no_sanitize("thread"))) +#else +#define AE_NO_TSAN +#endif +#else +#define AE_NO_TSAN +#endif + + +// AE_FORCEINLINE +#if defined(AE_VCPP) || defined(AE_ICC) +#define AE_FORCEINLINE __forceinline +#elif defined(AE_GCC) +//#define AE_FORCEINLINE __attribute__((always_inline)) +#define AE_FORCEINLINE inline +#else +#define AE_FORCEINLINE inline +#endif + + +// AE_ALIGN +#if defined(AE_VCPP) || defined(AE_ICC) +#define AE_ALIGN(x) __declspec(align(x)) +#elif defined(AE_GCC) +#define AE_ALIGN(x) __attribute__((aligned(x))) +#else +// Assume GCC compliant syntax... +#define AE_ALIGN(x) __attribute__((aligned(x))) +#endif + + +// Portable atomic fences implemented below: + +namespace moodycamel { + +enum memory_order { + memory_order_relaxed, + memory_order_acquire, + memory_order_release, + memory_order_acq_rel, + memory_order_seq_cst, + + // memory_order_sync: Forces a full sync: + // #LoadLoad, #LoadStore, #StoreStore, and most significantly, #StoreLoad + memory_order_sync = memory_order_seq_cst +}; + +} // end namespace moodycamel + +#if (defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli))) || (defined(AE_ICC) && __INTEL_COMPILER < 1600) +// VS2010 and ICC13 don't support std::atomic_*_fence, implement our own fences + +#include + +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) +#define AeFullSync _mm_mfence +#define AeLiteSync _mm_mfence +#elif defined(AE_ARCH_IA64) +#define AeFullSync __mf +#define AeLiteSync __mf +#elif defined(AE_ARCH_PPC) +#include +#define AeFullSync __sync +#define AeLiteSync __lwsync +#endif + + +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4365) // Disable erroneous 'conversion from long to unsigned int, signed/unsigned mismatch' error when using `assert` +#ifdef __cplusplus_cli +#pragma managed(push, off) +#endif +#endif + +namespace moodycamel { + +AE_FORCEINLINE void compiler_fence(memory_order order) AE_NO_TSAN +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: _ReadBarrier(); break; + case memory_order_release: _WriteBarrier(); break; + case memory_order_acq_rel: _ReadWriteBarrier(); break; + case memory_order_seq_cst: _ReadWriteBarrier(); break; + default: assert(false); + } +} + +// x86/x64 have a strong memory model -- all loads and stores have +// acquire and release semantics automatically (so only need compiler +// barriers for those). +#if defined(AE_ARCH_X86) || defined(AE_ARCH_X64) +AE_FORCEINLINE void fence(memory_order order) AE_NO_TSAN +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: _ReadBarrier(); break; + case memory_order_release: _WriteBarrier(); break; + case memory_order_acq_rel: _ReadWriteBarrier(); break; + case memory_order_seq_cst: + _ReadWriteBarrier(); + AeFullSync(); + _ReadWriteBarrier(); + break; + default: assert(false); + } +} +#else +AE_FORCEINLINE void fence(memory_order order) AE_NO_TSAN +{ + // Non-specialized arch, use heavier memory barriers everywhere just in case :-( + switch (order) { + case memory_order_relaxed: + break; + case memory_order_acquire: + _ReadBarrier(); + AeLiteSync(); + _ReadBarrier(); + break; + case memory_order_release: + _WriteBarrier(); + AeLiteSync(); + _WriteBarrier(); + break; + case memory_order_acq_rel: + _ReadWriteBarrier(); + AeLiteSync(); + _ReadWriteBarrier(); + break; + case memory_order_seq_cst: + _ReadWriteBarrier(); + AeFullSync(); + _ReadWriteBarrier(); + break; + default: assert(false); + } +} +#endif +} // end namespace moodycamel +#else +// Use standard library of atomics +#include + +namespace moodycamel { + +AE_FORCEINLINE void compiler_fence(memory_order order) AE_NO_TSAN +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: std::atomic_signal_fence(std::memory_order_acquire); break; + case memory_order_release: std::atomic_signal_fence(std::memory_order_release); break; + case memory_order_acq_rel: std::atomic_signal_fence(std::memory_order_acq_rel); break; + case memory_order_seq_cst: std::atomic_signal_fence(std::memory_order_seq_cst); break; + default: assert(false); + } +} + +AE_FORCEINLINE void fence(memory_order order) AE_NO_TSAN +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: std::atomic_thread_fence(std::memory_order_acquire); break; + case memory_order_release: std::atomic_thread_fence(std::memory_order_release); break; + case memory_order_acq_rel: std::atomic_thread_fence(std::memory_order_acq_rel); break; + case memory_order_seq_cst: std::atomic_thread_fence(std::memory_order_seq_cst); break; + default: assert(false); + } +} + +} // end namespace moodycamel + +#endif + + +#if !defined(AE_VCPP) || (_MSC_VER >= 1700 && !defined(__cplusplus_cli)) +#define AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC +#endif + +#ifdef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC +#include +#endif +#include + +// WARNING: *NOT* A REPLACEMENT FOR std::atomic. READ CAREFULLY: +// Provides basic support for atomic variables -- no memory ordering guarantees are provided. +// The guarantee of atomicity is only made for types that already have atomic load and store guarantees +// at the hardware level -- on most platforms this generally means aligned pointers and integers (only). +namespace moodycamel { +template +class weak_atomic +{ +public: + AE_NO_TSAN weak_atomic() : value() { } +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4100) // Get rid of (erroneous) 'unreferenced formal parameter' warning +#endif + template AE_NO_TSAN weak_atomic(U&& x) : value(std::forward(x)) { } +#ifdef __cplusplus_cli + // Work around bug with universal reference/nullptr combination that only appears when /clr is on + AE_NO_TSAN weak_atomic(nullptr_t) : value(nullptr) { } +#endif + AE_NO_TSAN weak_atomic(weak_atomic const& other) : value(other.load()) { } + AE_NO_TSAN weak_atomic(weak_atomic&& other) : value(std::move(other.load())) { } +#ifdef AE_VCPP +#pragma warning(pop) +#endif + + AE_FORCEINLINE operator T() const AE_NO_TSAN { return load(); } + + +#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC + template AE_FORCEINLINE weak_atomic const& operator=(U&& x) AE_NO_TSAN { value = std::forward(x); return *this; } + AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) AE_NO_TSAN { value = other.value; return *this; } + + AE_FORCEINLINE T load() const AE_NO_TSAN { return value; } + + AE_FORCEINLINE T fetch_add_acquire(T increment) AE_NO_TSAN + { +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) + if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment); +#if defined(_M_AMD64) + else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment); +#endif +#else +#error Unsupported platform +#endif + assert(false && "T must be either a 32 or 64 bit type"); + return value; + } + + AE_FORCEINLINE T fetch_add_release(T increment) AE_NO_TSAN + { +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) + if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment); +#if defined(_M_AMD64) + else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment); +#endif +#else +#error Unsupported platform +#endif + assert(false && "T must be either a 32 or 64 bit type"); + return value; + } +#else + template + AE_FORCEINLINE weak_atomic const& operator=(U&& x) AE_NO_TSAN + { + value.store(std::forward(x), std::memory_order_relaxed); + return *this; + } + + AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) AE_NO_TSAN + { + value.store(other.value.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + + AE_FORCEINLINE T load() const AE_NO_TSAN { return value.load(std::memory_order_relaxed); } + + AE_FORCEINLINE T fetch_add_acquire(T increment) AE_NO_TSAN + { + return value.fetch_add(increment, std::memory_order_acquire); + } + + AE_FORCEINLINE T fetch_add_release(T increment) AE_NO_TSAN + { + return value.fetch_add(increment, std::memory_order_release); + } +#endif + + +private: +#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC + // No std::atomic support, but still need to circumvent compiler optimizations. + // `volatile` will make memory access slow, but is guaranteed to be reliable. + volatile T value; +#else + std::atomic value; +#endif +}; + +} // end namespace moodycamel + + + +// Portable single-producer, single-consumer semaphore below: + +#if defined(_WIN32) +// Avoid including windows.h in a header; we only need a handful of +// items, so we'll redeclare them here (this is relatively safe since +// the API generally has to remain stable between Windows versions). +// I know this is an ugly hack but it still beats polluting the global +// namespace with thousands of generic names or adding a .cpp for nothing. +extern "C" { + struct _SECURITY_ATTRIBUTES; + __declspec(dllimport) void* __stdcall CreateSemaphoreW(_SECURITY_ATTRIBUTES* lpSemaphoreAttributes, long lInitialCount, long lMaximumCount, const wchar_t* lpName); + __declspec(dllimport) int __stdcall CloseHandle(void* hObject); + __declspec(dllimport) unsigned long __stdcall WaitForSingleObject(void* hHandle, unsigned long dwMilliseconds); + __declspec(dllimport) int __stdcall ReleaseSemaphore(void* hSemaphore, long lReleaseCount, long* lpPreviousCount); +} +#elif defined(__MACH__) +#include +#elif defined(__unix__) +#include +#endif + +namespace moodycamel +{ + // Code in the spsc_sema namespace below is an adaptation of Jeff Preshing's + // portable + lightweight semaphore implementations, originally from + // https://github.com/preshing/cpp11-on-multicore/blob/master/common/sema.h + // LICENSE: + // Copyright (c) 2015 Jeff Preshing + // + // This software is provided 'as-is', without any express or implied + // warranty. In no event will the authors be held liable for any damages + // arising from the use of this software. + // + // Permission is granted to anyone to use this software for any purpose, + // including commercial applications, and to alter it and redistribute it + // freely, subject to the following restrictions: + // + // 1. The origin of this software must not be misrepresented; you must not + // claim that you wrote the original software. If you use this software + // in a product, an acknowledgement in the product documentation would be + // appreciated but is not required. + // 2. Altered source versions must be plainly marked as such, and must not be + // misrepresented as being the original software. + // 3. This notice may not be removed or altered from any source distribution. + namespace spsc_sema + { +#if defined(_WIN32) + class Semaphore + { + private: + void* m_hSema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + AE_NO_TSAN Semaphore(int initialCount = 0) : m_hSema() + { + assert(initialCount >= 0); + const long maxLong = 0x7fffffff; + m_hSema = CreateSemaphoreW(nullptr, initialCount, maxLong, nullptr); + assert(m_hSema); + } + + AE_NO_TSAN ~Semaphore() + { + CloseHandle(m_hSema); + } + + bool wait() AE_NO_TSAN + { + const unsigned long infinite = 0xffffffff; + return WaitForSingleObject(m_hSema, infinite) == 0; + } + + bool try_wait() AE_NO_TSAN + { + return WaitForSingleObject(m_hSema, 0) == 0; + } + + bool timed_wait(std::uint64_t usecs) AE_NO_TSAN + { + return WaitForSingleObject(m_hSema, (unsigned long)(usecs / 1000)) == 0; + } + + void signal(int count = 1) AE_NO_TSAN + { + while (!ReleaseSemaphore(m_hSema, count, nullptr)); + } + }; +#elif defined(__MACH__) + //--------------------------------------------------------- + // Semaphore (Apple iOS and OSX) + // Can't use POSIX semaphores due to http://lists.apple.com/archives/darwin-kernel/2009/Apr/msg00010.html + //--------------------------------------------------------- + class Semaphore + { + private: + semaphore_t m_sema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + AE_NO_TSAN Semaphore(int initialCount = 0) : m_sema() + { + assert(initialCount >= 0); + kern_return_t rc = semaphore_create(mach_task_self(), &m_sema, SYNC_POLICY_FIFO, initialCount); + assert(rc == KERN_SUCCESS); + AE_UNUSED(rc); + } + + AE_NO_TSAN ~Semaphore() + { + semaphore_destroy(mach_task_self(), m_sema); + } + + bool wait() AE_NO_TSAN + { + return semaphore_wait(m_sema) == KERN_SUCCESS; + } + + bool try_wait() AE_NO_TSAN + { + return timed_wait(0); + } + + bool timed_wait(std::uint64_t timeout_usecs) AE_NO_TSAN + { + mach_timespec_t ts; + ts.tv_sec = static_cast(timeout_usecs / 1000000); + ts.tv_nsec = static_cast((timeout_usecs % 1000000) * 1000); + + // added in OSX 10.10: https://developer.apple.com/library/prerelease/mac/documentation/General/Reference/APIDiffsMacOSX10_10SeedDiff/modules/Darwin.html + kern_return_t rc = semaphore_timedwait(m_sema, ts); + return rc == KERN_SUCCESS; + } + + void signal() AE_NO_TSAN + { + while (semaphore_signal(m_sema) != KERN_SUCCESS); + } + + void signal(int count) AE_NO_TSAN + { + while (count-- > 0) + { + while (semaphore_signal(m_sema) != KERN_SUCCESS); + } + } + }; +#elif defined(__unix__) + //--------------------------------------------------------- + // Semaphore (POSIX, Linux) + //--------------------------------------------------------- + class Semaphore + { + private: + sem_t m_sema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + AE_NO_TSAN Semaphore(int initialCount = 0) : m_sema() + { + assert(initialCount >= 0); + int rc = sem_init(&m_sema, 0, static_cast(initialCount)); + assert(rc == 0); + AE_UNUSED(rc); + } + + AE_NO_TSAN ~Semaphore() + { + sem_destroy(&m_sema); + } + + bool wait() AE_NO_TSAN + { + // http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error + int rc; + do + { + rc = sem_wait(&m_sema); + } + while (rc == -1 && errno == EINTR); + return rc == 0; + } + + bool try_wait() AE_NO_TSAN + { + int rc; + do { + rc = sem_trywait(&m_sema); + } while (rc == -1 && errno == EINTR); + return rc == 0; + } + + bool timed_wait(std::uint64_t usecs) AE_NO_TSAN + { + struct timespec ts; + const int usecs_in_1_sec = 1000000; + const int nsecs_in_1_sec = 1000000000; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += static_cast(usecs / usecs_in_1_sec); + ts.tv_nsec += static_cast(usecs % usecs_in_1_sec) * 1000; + // sem_timedwait bombs if you have more than 1e9 in tv_nsec + // so we have to clean things up before passing it in + if (ts.tv_nsec >= nsecs_in_1_sec) { + ts.tv_nsec -= nsecs_in_1_sec; + ++ts.tv_sec; + } + + int rc; + do { + rc = sem_timedwait(&m_sema, &ts); + } while (rc == -1 && errno == EINTR); + return rc == 0; + } + + void signal() AE_NO_TSAN + { + while (sem_post(&m_sema) == -1); + } + + void signal(int count) AE_NO_TSAN + { + while (count-- > 0) + { + while (sem_post(&m_sema) == -1); + } + } + }; +#else +#error Unsupported platform! (No semaphore wrapper available) +#endif + + //--------------------------------------------------------- + // LightweightSemaphore + //--------------------------------------------------------- + class LightweightSemaphore + { + public: + typedef std::make_signed::type ssize_t; + + private: + weak_atomic m_count; + Semaphore m_sema; + + bool waitWithPartialSpinning(std::int64_t timeout_usecs = -1) AE_NO_TSAN + { + ssize_t oldCount; + // Is there a better way to set the initial spin count? + // If we lower it to 1000, testBenaphore becomes 15x slower on my Core i7-5930K Windows PC, + // as threads start hitting the kernel semaphore. + int spin = 1024; + while (--spin >= 0) + { + if (m_count.load() > 0) + { + m_count.fetch_add_acquire(-1); + return true; + } + compiler_fence(memory_order_acquire); // Prevent the compiler from collapsing the loop. + } + oldCount = m_count.fetch_add_acquire(-1); + if (oldCount > 0) + return true; + if (timeout_usecs < 0) + { + if (m_sema.wait()) + return true; + } + if (timeout_usecs > 0 && m_sema.timed_wait(static_cast(timeout_usecs))) + return true; + // At this point, we've timed out waiting for the semaphore, but the + // count is still decremented indicating we may still be waiting on + // it. So we have to re-adjust the count, but only if the semaphore + // wasn't signaled enough times for us too since then. If it was, we + // need to release the semaphore too. + while (true) + { + oldCount = m_count.fetch_add_release(1); + if (oldCount < 0) + return false; // successfully restored things to the way they were + // Oh, the producer thread just signaled the semaphore after all. Try again: + oldCount = m_count.fetch_add_acquire(-1); + if (oldCount > 0 && m_sema.try_wait()) + return true; + } + } + + public: + AE_NO_TSAN LightweightSemaphore(ssize_t initialCount = 0) : m_count(initialCount), m_sema() + { + assert(initialCount >= 0); + } + + bool tryWait() AE_NO_TSAN + { + if (m_count.load() > 0) + { + m_count.fetch_add_acquire(-1); + return true; + } + return false; + } + + bool wait() AE_NO_TSAN + { + return tryWait() || waitWithPartialSpinning(); + } + + bool wait(std::int64_t timeout_usecs) AE_NO_TSAN + { + return tryWait() || waitWithPartialSpinning(timeout_usecs); + } + + void signal(ssize_t count = 1) AE_NO_TSAN + { + assert(count >= 0); + ssize_t oldCount = m_count.fetch_add_release(count); + assert(oldCount >= -1); + if (oldCount < 0) + { + m_sema.signal(1); + } + } + + std::size_t availableApprox() const AE_NO_TSAN + { + ssize_t count = m_count.load(); + return count > 0 ? static_cast(count) : 0; + } + }; + } // end namespace spsc_sema +} // end namespace moodycamel + +#if defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli)) +#pragma warning(pop) +#ifdef __cplusplus_cli +#pragma managed(pop) +#endif +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/readerwritercircularbuffer.h b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/readerwritercircularbuffer.h new file mode 100644 index 0000000000000000000000000000000000000000..fc3ce962b5467017b32141a8f390e3c71776df6d --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/readerwritercircularbuffer.h @@ -0,0 +1,288 @@ +// ©2020 Cameron Desrochers. +// Distributed under the simplified BSD license (see the license file that +// should have come with this header). + +// Provides a C++11 implementation of a single-producer, single-consumer wait-free concurrent +// circular buffer (fixed-size queue). + +#pragma once + +#include +#include +#include +#include +#include +#include + +// Note that this implementation is fully modern C++11 (not compatible with old MSVC versions) +// but we still include atomicops.h for its LightweightSemaphore implementation. +#include "atomicops.h" + +#ifndef MOODYCAMEL_CACHE_LINE_SIZE +#define MOODYCAMEL_CACHE_LINE_SIZE 64 +#endif + +namespace moodycamel { + +template +class BlockingReaderWriterCircularBuffer +{ +public: + typedef T value_type; + +public: + explicit BlockingReaderWriterCircularBuffer(std::size_t capacity) + : maxcap(capacity), mask(), rawData(), data(), + slots_(new spsc_sema::LightweightSemaphore(static_cast(capacity))), + items(new spsc_sema::LightweightSemaphore(0)), + nextSlot(0), nextItem(0) + { + // Round capacity up to power of two to compute modulo mask. + // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + --capacity; + capacity |= capacity >> 1; + capacity |= capacity >> 2; + capacity |= capacity >> 4; + for (std::size_t i = 1; i < sizeof(std::size_t); i <<= 1) + capacity |= capacity >> (i << 3); + mask = capacity++; + rawData = static_cast(std::malloc(capacity * sizeof(T) + std::alignment_of::value - 1)); + data = align_for(rawData); + } + + BlockingReaderWriterCircularBuffer(BlockingReaderWriterCircularBuffer&& other) + : maxcap(0), mask(0), rawData(nullptr), data(nullptr), + slots_(new spsc_sema::LightweightSemaphore(0)), + items(new spsc_sema::LightweightSemaphore(0)), + nextSlot(), nextItem() + { + swap(other); + } + + BlockingReaderWriterCircularBuffer(BlockingReaderWriterCircularBuffer const&) = delete; + + // Note: The queue should not be accessed concurrently while it's + // being deleted. It's up to the user to synchronize this. + ~BlockingReaderWriterCircularBuffer() + { + for (std::size_t i = 0, n = items->availableApprox(); i != n; ++i) + reinterpret_cast(data)[(nextItem + i) & mask].~T(); + std::free(rawData); + } + + BlockingReaderWriterCircularBuffer& operator=(BlockingReaderWriterCircularBuffer&& other) noexcept + { + swap(other); + return *this; + } + + BlockingReaderWriterCircularBuffer& operator=(BlockingReaderWriterCircularBuffer const&) = delete; + + // Swaps the contents of this buffer with the contents of another. + // Not thread-safe. + void swap(BlockingReaderWriterCircularBuffer& other) noexcept + { + std::swap(maxcap, other.maxcap); + std::swap(mask, other.mask); + std::swap(rawData, other.rawData); + std::swap(data, other.data); + std::swap(slots_, other.slots_); + std::swap(items, other.items); + std::swap(nextSlot, other.nextSlot); + std::swap(nextItem, other.nextItem); + } + + // Enqueues a single item (by copying it). + // Fails if not enough room to enqueue. + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + bool try_enqueue(T const& item) + { + if (!slots_->tryWait()) + return false; + inner_enqueue(item); + return true; + } + + // Enqueues a single item (by moving it, if possible). + // Fails if not enough room to enqueue. + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + bool try_enqueue(T&& item) + { + if (!slots_->tryWait()) + return false; + inner_enqueue(std::move(item)); + return true; + } + + // Blocks the current thread until there's enough space to enqueue the given item, + // then enqueues it (via copy). + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + void wait_enqueue(T const& item) + { + while (!slots_->wait()); + inner_enqueue(item); + } + + // Blocks the current thread until there's enough space to enqueue the given item, + // then enqueues it (via move, if possible). + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + void wait_enqueue(T&& item) + { + while (!slots_->wait()); + inner_enqueue(std::move(item)); + } + + // Blocks the current thread until there's enough space to enqueue the given item, + // or the timeout expires. Returns false without enqueueing the item if the timeout + // expires, otherwise enqueues the item (via copy) and returns true. + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + bool wait_enqueue_timed(T const& item, std::int64_t timeout_usecs) + { + if (!slots_->wait(timeout_usecs)) + return false; + inner_enqueue(item); + return true; + } + + // Blocks the current thread until there's enough space to enqueue the given item, + // or the timeout expires. Returns false without enqueueing the item if the timeout + // expires, otherwise enqueues the item (via move, if possible) and returns true. + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + bool wait_enqueue_timed(T&& item, std::int64_t timeout_usecs) + { + if (!slots_->wait(timeout_usecs)) + return false; + inner_enqueue(std::move(item)); + return true; + } + + // Blocks the current thread until there's enough space to enqueue the given item, + // or the timeout expires. Returns false without enqueueing the item if the timeout + // expires, otherwise enqueues the item (via copy) and returns true. + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + template + inline bool wait_enqueue_timed(T const& item, std::chrono::duration const& timeout) + { + return wait_enqueue_timed(item, std::chrono::duration_cast(timeout).count()); + } + + // Blocks the current thread until there's enough space to enqueue the given item, + // or the timeout expires. Returns false without enqueueing the item if the timeout + // expires, otherwise enqueues the item (via move, if possible) and returns true. + // Thread-safe when called by producer thread. + // No exception guarantee (state will be corrupted) if constructor of T throws. + template + inline bool wait_enqueue_timed(T&& item, std::chrono::duration const& timeout) + { + return wait_enqueue_timed(std::move(item), std::chrono::duration_cast(timeout).count()); + } + + // Attempts to dequeue a single item. + // Returns false if the buffer is empty. + // Thread-safe when called by consumer thread. + // No exception guarantee (state will be corrupted) if assignment operator of U throws. + template + bool try_dequeue(U& item) + { + if (!items->tryWait()) + return false; + inner_dequeue(item); + return true; + } + + // Blocks the current thread until there's something to dequeue, then dequeues it. + // Thread-safe when called by consumer thread. + // No exception guarantee (state will be corrupted) if assignment operator of U throws. + template + void wait_dequeue(U& item) + { + while (!items->wait()); + inner_dequeue(item); + } + + // Blocks the current thread until either there's something to dequeue + // or the timeout expires. Returns false without setting `item` if the + // timeout expires, otherwise assigns to `item` and returns true. + // Thread-safe when called by consumer thread. + // No exception guarantee (state will be corrupted) if assignment operator of U throws. + template + bool wait_dequeue_timed(U& item, std::int64_t timeout_usecs) + { + if (!items->wait(timeout_usecs)) + return false; + inner_dequeue(item); + return true; + } + + // Blocks the current thread until either there's something to dequeue + // or the timeout expires. Returns false without setting `item` if the + // timeout expires, otherwise assigns to `item` and returns true. + // Thread-safe when called by consumer thread. + // No exception guarantee (state will be corrupted) if assignment operator of U throws. + template + inline bool wait_dequeue_timed(U& item, std::chrono::duration const& timeout) + { + return wait_dequeue_timed(item, std::chrono::duration_cast(timeout).count()); + } + + // Returns a (possibly outdated) snapshot of the total number of elements currently in the buffer. + // Thread-safe. + inline std::size_t size_approx() const + { + return items->availableApprox(); + } + + // Returns the maximum number of elements that this circular buffer can hold at once. + // Thread-safe. + inline std::size_t max_capacity() const + { + return maxcap; + } + +private: + template + void inner_enqueue(U&& item) + { + std::size_t i = nextSlot++; + new (reinterpret_cast(data) + (i & mask)) T(std::forward(item)); + items->signal(); + } + + template + void inner_dequeue(U& item) + { + std::size_t i = nextItem++; + T& element = reinterpret_cast(data)[i & mask]; + item = std::move(element); + element.~T(); + slots_->signal(); + } + + template + static inline char* align_for(char* ptr) + { + const std::size_t alignment = std::alignment_of::value; + return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; + } + +private: + std::size_t maxcap; // actual (non-power-of-two) capacity + std::size_t mask; // circular buffer capacity mask (for cheap modulo) + char* rawData; // raw circular buffer memory + char* data; // circular buffer memory aligned to element alignment + std::unique_ptr slots_; // number of slots_ currently free + std::unique_ptr items; // number of elements currently enqueued + char cachelineFiller0[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(char*) * 2 - sizeof(std::size_t) * 2 - sizeof(std::unique_ptr) * 2]; + std::size_t nextSlot; // index of next free slot to enqueue into + char cachelineFiller1[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(std::size_t)]; + std::size_t nextItem; // index of next element to dequeue from +}; + +} diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/readerwriterqueue.h b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/readerwriterqueue.h new file mode 100644 index 0000000000000000000000000000000000000000..d87110a32062f0699f0f4c6b1625ee0df7184bed --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/moodycamel/spsc/readerwriterqueue.h @@ -0,0 +1,979 @@ +// ©2013-2020 Cameron Desrochers. +// Distributed under the simplified BSD license (see the license file that +// should have come with this header). + +#pragma once + +#include "atomicops.h" +#include +#include +#include +#include +#include +#include +#include +#include // For malloc/free/abort & size_t +#include +#if __cplusplus > 199711L || _MSC_VER >= 1700 // C++11 or VS2012 +#include +#endif + + +// A lock-free queue for a single-consumer, single-producer architecture. +// The queue is also wait-free in the common path (except if more memory +// needs to be allocated, in which case malloc is called). +// Allocates memory sparingly, and only once if the original maximum size +// estimate is never exceeded. +// Tested on x86/x64 processors, but semantics should be correct for all +// architectures (given the right implementations in atomicops.h), provided +// that aligned integer and pointer accesses are naturally atomic. +// Note that there should only be one consumer thread and producer thread; +// Switching roles of the threads, or using multiple consecutive threads for +// one role, is not safe unless properly synchronized. +// Using the queue exclusively from one thread is fine, though a bit silly. + +#ifndef MOODYCAMEL_CACHE_LINE_SIZE +#define MOODYCAMEL_CACHE_LINE_SIZE 64 +#endif + +#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED +#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__)) +#define MOODYCAMEL_EXCEPTIONS_ENABLED +#endif +#endif + +#ifndef MOODYCAMEL_HAS_EMPLACE +#if !defined(_MSC_VER) || _MSC_VER >= 1800 // variadic templates: either a non-MS compiler or VS >= 2013 +#define MOODYCAMEL_HAS_EMPLACE 1 +#endif +#endif + +#ifndef MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE +#if defined (__APPLE__) && defined (__MACH__) && __cplusplus >= 201703L +// This is required to find out what deployment target we are using +#include +#if !defined(MAC_OS_X_VERSION_MIN_REQUIRED) || MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_14 +// C++17 new(size_t, align_val_t) is not backwards-compatible with older versions of macOS, so we can't support over-alignment in this case +#define MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE +#endif +#endif +#endif + +#ifndef MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE +#define MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE AE_ALIGN(MOODYCAMEL_CACHE_LINE_SIZE) +#endif + +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4324) // structure was padded due to __declspec(align()) +#pragma warning(disable: 4820) // padding was added +#pragma warning(disable: 4127) // conditional expression is constant +#endif + +namespace moodycamel { + +template +class MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE ReaderWriterQueue +{ + // Design: Based on a queue-of-queues. The low-level queues are just + // circular buffers with front and tail indices indicating where the + // next element to dequeue is and where the next element can be enqueued, + // respectively. Each low-level queue is called a "block". Each block + // wastes exactly one element's worth of space to keep the design simple + // (if front == tail then the queue is empty, and can't be full). + // The high-level queue is a circular linked list of blocks; again there + // is a front and tail, but this time they are pointers to the blocks. + // The front block is where the next element to be dequeued is, provided + // the block is not empty. The back block is where elements are to be + // enqueued, provided the block is not full. + // The producer thread owns all the tail indices/pointers. The consumer + // thread owns all the front indices/pointers. Both threads read each + // other's variables, but only the owning thread updates them. E.g. After + // the consumer reads the producer's tail, the tail may change before the + // consumer is done dequeuing an object, but the consumer knows the tail + // will never go backwards, only forwards. + // If there is no room to enqueue an object, an additional block (of + // equal size to the last block) is added. Blocks are never removed. + +public: + typedef T value_type; + + // Constructs a queue that can hold at least `size` elements without further + // allocations. If more than MAX_BLOCK_SIZE elements are requested, + // then several blocks of MAX_BLOCK_SIZE each are reserved (including + // at least one extra buffer block). + AE_NO_TSAN explicit ReaderWriterQueue(size_t size = 15) +#ifndef NDEBUG + : enqueuing(false) + ,dequeuing(false) +#endif + { + assert(MAX_BLOCK_SIZE == ceilToPow2(MAX_BLOCK_SIZE) && "MAX_BLOCK_SIZE must be a power of 2"); + assert(MAX_BLOCK_SIZE >= 2 && "MAX_BLOCK_SIZE must be at least 2"); + + Block* firstBlock = nullptr; + + largestBlockSize = ceilToPow2(size + 1); // We need a spare slot to fit size elements in the block + if (largestBlockSize > MAX_BLOCK_SIZE * 2) { + // We need a spare block in case the producer is writing to a different block the consumer is reading from, and + // wants to enqueue the maximum number of elements. We also need a spare element in each block to avoid the ambiguity + // between front == tail meaning "empty" and "full". + // So the effective number of slots that are guaranteed to be usable at any time is the block size - 1 times the + // number of blocks - 1. Solving for size and applying a ceiling to the division gives us (after simplifying): + size_t initialBlockCount = (size + MAX_BLOCK_SIZE * 2 - 3) / (MAX_BLOCK_SIZE - 1); + largestBlockSize = MAX_BLOCK_SIZE; + Block* lastBlock = nullptr; + for (size_t i = 0; i != initialBlockCount; ++i) { + auto block = make_block(largestBlockSize); + if (block == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED + throw std::bad_alloc(); +#else + abort(); +#endif + } + if (firstBlock == nullptr) { + firstBlock = block; + } + else { + lastBlock->next = block; + } + lastBlock = block; + block->next = firstBlock; + } + } + else { + firstBlock = make_block(largestBlockSize); + if (firstBlock == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED + throw std::bad_alloc(); +#else + abort(); +#endif + } + firstBlock->next = firstBlock; + } + frontBlock = firstBlock; + tailBlock = firstBlock; + + // Make sure the reader/writer threads will have the initialized memory setup above: + fence(memory_order_sync); + } + + // Note: The queue should not be accessed concurrently while it's + // being moved. It's up to the user to synchronize this. + AE_NO_TSAN ReaderWriterQueue(ReaderWriterQueue&& other) + : frontBlock(other.frontBlock.load()), + tailBlock(other.tailBlock.load()), + largestBlockSize(other.largestBlockSize) +#ifndef NDEBUG + ,enqueuing(false) + ,dequeuing(false) +#endif + { + other.largestBlockSize = 32; + Block* b = other.make_block(other.largestBlockSize); + if (b == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED + throw std::bad_alloc(); +#else + abort(); +#endif + } + b->next = b; + other.frontBlock = b; + other.tailBlock = b; + } + + // Note: The queue should not be accessed concurrently while it's + // being moved. It's up to the user to synchronize this. + ReaderWriterQueue& operator=(ReaderWriterQueue&& other) AE_NO_TSAN + { + Block* b = frontBlock.load(); + frontBlock = other.frontBlock.load(); + other.frontBlock = b; + b = tailBlock.load(); + tailBlock = other.tailBlock.load(); + other.tailBlock = b; + std::swap(largestBlockSize, other.largestBlockSize); + return *this; + } + + // Note: The queue should not be accessed concurrently while it's + // being deleted. It's up to the user to synchronize this. + AE_NO_TSAN ~ReaderWriterQueue() + { + // Make sure we get the latest version of all variables from other CPUs: + fence(memory_order_sync); + + // Destroy any remaining objects in queue and free memory + Block* frontBlock_ = frontBlock; + Block* block = frontBlock_; + do { + Block* nextBlock = block->next; + size_t blockFront = block->front; + size_t blockTail = block->tail; + + for (size_t i = blockFront; i != blockTail; i = (i + 1) & block->sizeMask) { + auto element = reinterpret_cast(block->data + i * sizeof(T)); + element->~T(); + (void)element; + } + + auto rawBlock = block->rawThis; + block->~Block(); + std::free(rawBlock); + block = nextBlock; + } while (block != frontBlock_); + } + + + // Enqueues a copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T const& element) AE_NO_TSAN + { + return inner_enqueue(element); + } + + // Enqueues a moved copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T&& element) AE_NO_TSAN + { + return inner_enqueue(std::forward(element)); + } + +#if MOODYCAMEL_HAS_EMPLACE + // Like try_enqueue() but with emplace semantics (i.e. construct-in-place). + template + AE_FORCEINLINE bool try_emplace(Args&&... args) AE_NO_TSAN + { + return inner_enqueue(std::forward(args)...); + } +#endif + + // Enqueues a copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T const& element) AE_NO_TSAN + { + return inner_enqueue(element); + } + + // Enqueues a moved copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T&& element) AE_NO_TSAN + { + return inner_enqueue(std::forward(element)); + } + +#if MOODYCAMEL_HAS_EMPLACE + // Like enqueue() but with emplace semantics (i.e. construct-in-place). + template + AE_FORCEINLINE bool emplace(Args&&... args) AE_NO_TSAN + { + return inner_enqueue(std::forward(args)...); + } +#endif + + // Attempts to dequeue an element; if the queue is empty, + // returns false instead. If the queue has at least one element, + // moves front to result using operator=, then returns true. + template + bool try_dequeue(U& result) AE_NO_TSAN + { +#ifndef NDEBUG + ReentrantGuard guard(this->dequeuing); +#endif + + // High-level pseudocode: + // Remember where the tail block is + // If the front block has an element in it, dequeue it + // Else + // If front block was the tail block when we entered the function, return false + // Else advance to next block and dequeue the item there + + // Note that we have to use the value of the tail block from before we check if the front + // block is full or not, in case the front block is empty and then, before we check if the + // tail block is at the front block or not, the producer fills up the front block *and + // moves on*, which would make us skip a filled block. Seems unlikely, but was consistently + // reproducible in practice. + // In order to avoid overhead in the common case, though, we do a double-checked pattern + // where we have the fast path if the front block is not empty, then read the tail block, + // then re-read the front block and check if it's not empty again, then check if the tail + // block has advanced. + + Block* frontBlock_ = frontBlock.load(); + size_t blockTail = frontBlock_->localTail; + size_t blockFront = frontBlock_->front.load(); + + if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { + fence(memory_order_acquire); + + non_empty_front_block: + // Front block not empty, dequeue from here + auto element = reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); + result = std::move(*element); + element->~T(); + + blockFront = (blockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = blockFront; + } + else if (frontBlock_ != tailBlock.load()) { + fence(memory_order_acquire); + + frontBlock_ = frontBlock.load(); + blockTail = frontBlock_->localTail = frontBlock_->tail.load(); + blockFront = frontBlock_->front.load(); + fence(memory_order_acquire); + + if (blockFront != blockTail) { + // Oh look, the front block isn't empty after all + goto non_empty_front_block; + } + + // Front block is empty but there's another block ahead, advance to it + Block* nextBlock = frontBlock_->next; + // Don't need an acquire fence here since next can only ever be set on the tailBlock, + // and we're not the tailBlock, and we did an acquire earlier after reading tailBlock which + // ensures next is up-to-date on this CPU in case we recently were at tailBlock. + + size_t nextBlockFront = nextBlock->front.load(); + size_t nextBlockTail = nextBlock->localTail = nextBlock->tail.load(); + fence(memory_order_acquire); + + // Since the tailBlock is only ever advanced after being written to, + // we know there's for sure an element to dequeue on it + assert(nextBlockFront != nextBlockTail); + AE_UNUSED(nextBlockTail); + + // We're done with this block, let the producer use it if it needs + fence(memory_order_release); // Expose possibly pending changes to frontBlock->front from last dequeue + frontBlock = frontBlock_ = nextBlock; + + compiler_fence(memory_order_release); // Not strictly needed + + auto element = reinterpret_cast(frontBlock_->data + nextBlockFront * sizeof(T)); + + result = std::move(*element); + element->~T(); + + nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = nextBlockFront; + } + else { + // No elements in current block and no other block to advance to + return false; + } + + return true; + } + + + // Returns a pointer to the front element in the queue (the one that + // would be removed next by a call to `try_dequeue` or `pop`). If the + // queue appears empty at the time the method is called, nullptr is + // returned instead. + // Must be called only from the consumer thread. + T* peek() const AE_NO_TSAN + { +#ifndef NDEBUG + ReentrantGuard guard(this->dequeuing); +#endif + // See try_dequeue() for reasoning + + Block* frontBlock_ = frontBlock.load(); + size_t blockTail = frontBlock_->localTail; + size_t blockFront = frontBlock_->front.load(); + + if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { + fence(memory_order_acquire); + non_empty_front_block: + return reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); + } + else if (frontBlock_ != tailBlock.load()) { + fence(memory_order_acquire); + frontBlock_ = frontBlock.load(); + blockTail = frontBlock_->localTail = frontBlock_->tail.load(); + blockFront = frontBlock_->front.load(); + fence(memory_order_acquire); + + if (blockFront != blockTail) { + goto non_empty_front_block; + } + + Block* nextBlock = frontBlock_->next; + + size_t nextBlockFront = nextBlock->front.load(); + fence(memory_order_acquire); + + assert(nextBlockFront != nextBlock->tail.load()); + return reinterpret_cast(nextBlock->data + nextBlockFront * sizeof(T)); + } + + return nullptr; + } + + // Removes the front element from the queue, if any, without returning it. + // Returns true on success, or false if the queue appeared empty at the time + // `pop` was called. + bool pop() AE_NO_TSAN + { +#ifndef NDEBUG + ReentrantGuard guard(this->dequeuing); +#endif + // See try_dequeue() for reasoning + + Block* frontBlock_ = frontBlock.load(); + size_t blockTail = frontBlock_->localTail; + size_t blockFront = frontBlock_->front.load(); + + if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { + fence(memory_order_acquire); + + non_empty_front_block: + auto element = reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); + element->~T(); + + blockFront = (blockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = blockFront; + } + else if (frontBlock_ != tailBlock.load()) { + fence(memory_order_acquire); + frontBlock_ = frontBlock.load(); + blockTail = frontBlock_->localTail = frontBlock_->tail.load(); + blockFront = frontBlock_->front.load(); + fence(memory_order_acquire); + + if (blockFront != blockTail) { + goto non_empty_front_block; + } + + // Front block is empty but there's another block ahead, advance to it + Block* nextBlock = frontBlock_->next; + + size_t nextBlockFront = nextBlock->front.load(); + size_t nextBlockTail = nextBlock->localTail = nextBlock->tail.load(); + fence(memory_order_acquire); + + assert(nextBlockFront != nextBlockTail); + AE_UNUSED(nextBlockTail); + + fence(memory_order_release); + frontBlock = frontBlock_ = nextBlock; + + compiler_fence(memory_order_release); + + auto element = reinterpret_cast(frontBlock_->data + nextBlockFront * sizeof(T)); + element->~T(); + + nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = nextBlockFront; + } + else { + // No elements in current block and no other block to advance to + return false; + } + + return true; + } + + // Returns the approximate number of items currently in the queue. + // Safe to call from both the producer and consumer threads. + inline size_t size_approx() const AE_NO_TSAN + { + size_t result = 0; + Block* frontBlock_ = frontBlock.load(); + Block* block = frontBlock_; + do { + fence(memory_order_acquire); + size_t blockFront = block->front.load(); + size_t blockTail = block->tail.load(); + result += (blockTail - blockFront) & block->sizeMask; + block = block->next.load(); + } while (block != frontBlock_); + return result; + } + + // Returns the total number of items that could be enqueued without incurring + // an allocation when this queue is empty. + // Safe to call from both the producer and consumer threads. + // + // NOTE: The actual capacity during usage may be different depending on the consumer. + // If the consumer is removing elements concurrently, the producer cannot add to + // the block the consumer is removing from until it's completely empty, except in + // the case where the producer was writing to the same block the consumer was + // reading from the whole time. + inline size_t max_capacity() const { + size_t result = 0; + Block* frontBlock_ = frontBlock.load(); + Block* block = frontBlock_; + do { + fence(memory_order_acquire); + result += block->sizeMask; + block = block->next.load(); + } while (block != frontBlock_); + return result; + } + + +private: + enum AllocationMode { CanAlloc, CannotAlloc }; + +#if MOODYCAMEL_HAS_EMPLACE + template + bool inner_enqueue(Args&&... args) AE_NO_TSAN +#else + template + bool inner_enqueue(U&& element) AE_NO_TSAN +#endif + { +#ifndef NDEBUG + ReentrantGuard guard(this->enqueuing); +#endif + + // High-level pseudocode (assuming we're allowed to alloc a new block): + // If room in tail block, add to tail + // Else check next block + // If next block is not the head block, enqueue on next block + // Else create a new block and enqueue there + // Advance tail to the block we just enqueued to + + Block* tailBlock_ = tailBlock.load(); + size_t blockFront = tailBlock_->localFront; + size_t blockTail = tailBlock_->tail.load(); + + size_t nextBlockTail = (blockTail + 1) & tailBlock_->sizeMask; + if (nextBlockTail != blockFront || nextBlockTail != (tailBlock_->localFront = tailBlock_->front.load())) { + fence(memory_order_acquire); + // This block has room for at least one more element + char* location = tailBlock_->data + blockTail * sizeof(T); +#if MOODYCAMEL_HAS_EMPLACE + new (location) T(std::forward(args)...); +#else + new (location) T(std::forward(element)); +#endif + + fence(memory_order_release); + tailBlock_->tail = nextBlockTail; + } + else { + fence(memory_order_acquire); + if (tailBlock_->next.load() != frontBlock) { + // Note that the reason we can't advance to the frontBlock and start adding new entries there + // is because if we did, then dequeue would stay in that block, eventually reading the new values, + // instead of advancing to the next full block (whose values were enqueued first and so should be + // consumed first). + + fence(memory_order_acquire); // Ensure we get latest writes if we got the latest frontBlock + + // tailBlock is full, but there's a free block ahead, use it + Block* tailBlockNext = tailBlock_->next.load(); + size_t nextBlockFront = tailBlockNext->localFront = tailBlockNext->front.load(); + nextBlockTail = tailBlockNext->tail.load(); + fence(memory_order_acquire); + + // This block must be empty since it's not the head block and we + // go through the blocks in a circle + assert(nextBlockFront == nextBlockTail); + tailBlockNext->localFront = nextBlockFront; + + char* location = tailBlockNext->data + nextBlockTail * sizeof(T); +#if MOODYCAMEL_HAS_EMPLACE + new (location) T(std::forward(args)...); +#else + new (location) T(std::forward(element)); +#endif + + tailBlockNext->tail = (nextBlockTail + 1) & tailBlockNext->sizeMask; + + fence(memory_order_release); + tailBlock = tailBlockNext; + } + else if (canAlloc == CanAlloc) { + // tailBlock is full and there's no free block ahead; create a new block + auto newBlockSize = largestBlockSize >= MAX_BLOCK_SIZE ? largestBlockSize : largestBlockSize * 2; + auto newBlock = make_block(newBlockSize); + if (newBlock == nullptr) { + // Could not allocate a block! + return false; + } + largestBlockSize = newBlockSize; + +#if MOODYCAMEL_HAS_EMPLACE + new (newBlock->data) T(std::forward(args)...); +#else + new (newBlock->data) T(std::forward(element)); +#endif + assert(newBlock->front == 0); + newBlock->tail = newBlock->localTail = 1; + + newBlock->next = tailBlock_->next.load(); + tailBlock_->next = newBlock; + + // Might be possible for the dequeue thread to see the new tailBlock->next + // *without* seeing the new tailBlock value, but this is OK since it can't + // advance to the next block until tailBlock is set anyway (because the only + // case where it could try to read the next is if it's already at the tailBlock, + // and it won't advance past tailBlock in any circumstance). + + fence(memory_order_release); + tailBlock = newBlock; + } + else if (canAlloc == CannotAlloc) { + // Would have had to allocate a new block to enqueue, but not allowed + return false; + } + else { + assert(false && "Should be unreachable code"); + return false; + } + } + + return true; + } + + + // Disable copying + ReaderWriterQueue(ReaderWriterQueue const&) { } + + // Disable assignment + ReaderWriterQueue& operator=(ReaderWriterQueue const&) { } + + + AE_FORCEINLINE static size_t ceilToPow2(size_t x) + { + // From http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + --x; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + for (size_t i = 1; i < sizeof(size_t); i <<= 1) { + x |= x >> (i << 3); + } + ++x; + return x; + } + + template + static AE_FORCEINLINE char* align_for(char* ptr) AE_NO_TSAN + { + const std::size_t alignment = std::alignment_of::value; + return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; + } +private: +#ifndef NDEBUG + struct ReentrantGuard + { + AE_NO_TSAN ReentrantGuard(weak_atomic& _inSection) + : inSection(_inSection) + { + assert(!inSection && "Concurrent (or re-entrant) enqueue or dequeue operation detected (only one thread at a time may hold the producer or consumer role)"); + inSection = true; + } + + AE_NO_TSAN ~ReentrantGuard() { inSection = false; } + + private: + ReentrantGuard& operator=(ReentrantGuard const&); + + private: + weak_atomic& inSection; + }; +#endif + + struct Block + { + // Avoid false-sharing by putting highly contended variables on their own cache lines + weak_atomic front; // (Atomic) Elements are read from here + size_t localTail; // An uncontended shadow copy of tail, owned by the consumer + + char cachelineFiller0[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic) - sizeof(size_t)]; + weak_atomic tail; // (Atomic) Elements are enqueued here + size_t localFront; + + char cachelineFiller1[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic) - sizeof(size_t)]; // next isn't very contended, but we don't want it on the same cache line as tail (which is) + weak_atomic next; // (Atomic) + + char* data; // Contents (on heap) are aligned to T's alignment + + const size_t sizeMask; + + + // size must be a power of two (and greater than 0) + AE_NO_TSAN Block(size_t const& _size, char* _rawThis, char* _data) + : front(0UL), localTail(0), tail(0UL), localFront(0), next(nullptr), data(_data), sizeMask(_size - 1), rawThis(_rawThis) + { + } + + private: + // C4512 - Assignment operator could not be generated + Block& operator=(Block const&); + + public: + char* rawThis; + }; + + + static Block* make_block(size_t capacity) AE_NO_TSAN + { + // Allocate enough memory for the block itself, as well as all the elements it will contain + auto size = sizeof(Block) + std::alignment_of::value - 1; + size += sizeof(T) * capacity + std::alignment_of::value - 1; + auto newBlockRaw = static_cast(std::malloc(size)); + if (newBlockRaw == nullptr) { + return nullptr; + } + + auto newBlockAligned = align_for(newBlockRaw); + auto newBlockData = align_for(newBlockAligned + sizeof(Block)); + return new (newBlockAligned) Block(capacity, newBlockRaw, newBlockData); + } + +private: + weak_atomic frontBlock; // (Atomic) Elements are dequeued from this block + + char cachelineFiller[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic)]; + weak_atomic tailBlock; // (Atomic) Elements are enqueued to this block + + size_t largestBlockSize; + +#ifndef NDEBUG + weak_atomic enqueuing; + mutable weak_atomic dequeuing; +#endif +}; + +// Like ReaderWriterQueue, but also providees blocking operations +template +class BlockingReaderWriterQueue +{ +private: + typedef ::moodycamel::ReaderWriterQueue ReaderWriterQueue; + +public: + explicit BlockingReaderWriterQueue(size_t size = 15) AE_NO_TSAN + : inner(size), sema(new spsc_sema::LightweightSemaphore()) + { } + + BlockingReaderWriterQueue(BlockingReaderWriterQueue&& other) AE_NO_TSAN + : inner(std::move(other.inner)), sema(std::move(other.sema)) + { } + + BlockingReaderWriterQueue& operator=(BlockingReaderWriterQueue&& other) AE_NO_TSAN + { + std::swap(sema, other.sema); + std::swap(inner, other.inner); + return *this; + } + + + // Enqueues a copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T const& element) AE_NO_TSAN + { + if (inner.try_enqueue(element)) { + sema->signal(); + return true; + } + return false; + } + + // Enqueues a moved copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T&& element) AE_NO_TSAN + { + if (inner.try_enqueue(std::forward(element))) { + sema->signal(); + return true; + } + return false; + } + +#if MOODYCAMEL_HAS_EMPLACE + // Like try_enqueue() but with emplace semantics (i.e. construct-in-place). + template + AE_FORCEINLINE bool try_emplace(Args&&... args) AE_NO_TSAN + { + if (inner.try_emplace(std::forward(args)...)) { + sema->signal(); + return true; + } + return false; + } +#endif + + + // Enqueues a copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T const& element) AE_NO_TSAN + { + if (inner.enqueue(element)) { + sema->signal(); + return true; + } + return false; + } + + // Enqueues a moved copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T&& element) AE_NO_TSAN + { + if (inner.enqueue(std::forward(element))) { + sema->signal(); + return true; + } + return false; + } + +#if MOODYCAMEL_HAS_EMPLACE + // Like enqueue() but with emplace semantics (i.e. construct-in-place). + template + AE_FORCEINLINE bool emplace(Args&&... args) AE_NO_TSAN + { + if (inner.emplace(std::forward(args)...)) { + sema->signal(); + return true; + } + return false; + } +#endif + + + // Attempts to dequeue an element; if the queue is empty, + // returns false instead. If the queue has at least one element, + // moves front to result using operator=, then returns true. + template + bool try_dequeue(U& result) AE_NO_TSAN + { + if (sema->tryWait()) { + bool success = inner.try_dequeue(result); + assert(success); + AE_UNUSED(success); + return true; + } + return false; + } + + + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available, then dequeues it. + template + void wait_dequeue(U& result) AE_NO_TSAN + { + while (!sema->wait()); + bool success = inner.try_dequeue(result); + AE_UNUSED(result); + assert(success); + AE_UNUSED(success); + } + + + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available up to the specified timeout, + // then dequeues it and returns true, or returns false if the timeout + // expires before an element can be dequeued. + // Using a negative timeout indicates an indefinite timeout, + // and is thus functionally equivalent to calling wait_dequeue. + template + bool wait_dequeue_timed(U& result, std::int64_t timeout_usecs) AE_NO_TSAN + { + if (!sema->wait(timeout_usecs)) { + return false; + } + bool success = inner.try_dequeue(result); + AE_UNUSED(result); + assert(success); + AE_UNUSED(success); + return true; + } + + +#if __cplusplus > 199711L || _MSC_VER >= 1700 + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available up to the specified timeout, + // then dequeues it and returns true, or returns false if the timeout + // expires before an element can be dequeued. + // Using a negative timeout indicates an indefinite timeout, + // and is thus functionally equivalent to calling wait_dequeue. + template + inline bool wait_dequeue_timed(U& result, std::chrono::duration const& timeout) AE_NO_TSAN + { + return wait_dequeue_timed(result, std::chrono::duration_cast(timeout).count()); + } +#endif + + + // Returns a pointer to the front element in the queue (the one that + // would be removed next by a call to `try_dequeue` or `pop`). If the + // queue appears empty at the time the method is called, nullptr is + // returned instead. + // Must be called only from the consumer thread. + AE_FORCEINLINE T* peek() const AE_NO_TSAN + { + return inner.peek(); + } + + // Removes the front element from the queue, if any, without returning it. + // Returns true on success, or false if the queue appeared empty at the time + // `pop` was called. + AE_FORCEINLINE bool pop() AE_NO_TSAN + { + if (sema->tryWait()) { + bool result = inner.pop(); + assert(result); + AE_UNUSED(result); + return true; + } + return false; + } + + // Returns the approximate number of items currently in the queue. + // Safe to call from both the producer and consumer threads. + AE_FORCEINLINE size_t size_approx() const AE_NO_TSAN + { + return sema->availableApprox(); + } + + // Returns the total number of items that could be enqueued without incurring + // an allocation when this queue is empty. + // Safe to call from both the producer and consumer threads. + // + // NOTE: The actual capacity during usage may be different depending on the consumer. + // If the consumer is removing elements concurrently, the producer cannot add to + // the block the consumer is removing from until it's completely empty, except in + // the case where the producer was writing to the same block the consumer was + // reading from the whole time. + AE_FORCEINLINE size_t max_capacity() const { + return inner.max_capacity(); + } + +private: + // Disable copying & assignment + BlockingReaderWriterQueue(BlockingReaderWriterQueue const&) { } + BlockingReaderWriterQueue& operator=(BlockingReaderWriterQueue const&) { } + +private: + ReaderWriterQueue inner; + std::unique_ptr sema; +}; + +} // end namespace moodycamel + +#ifdef AE_VCPP +#pragma warning(pop) +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/lib/oes/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d8a887b5b289551e4564f9be7ed9f32b31daec51 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/CMakeLists.txt @@ -0,0 +1,6 @@ +add_subdirectory(core) +add_subdirectory(eml) + +add_subdirectory(postgresql) + +add_subdirectory(testing) diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1b391d3576b9ab289a10f8e7d758be835f22e1df --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(utils) +add_subdirectory(http) \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/crypto/OpenSSLCompat.hpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/crypto/OpenSSLCompat.hpp new file mode 100644 index 0000000000000000000000000000000000000000..6066ca7393887b18bcdcd7abde6ea32cde0ca6c6 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/crypto/OpenSSLCompat.hpp @@ -0,0 +1,60 @@ +// ============================================================================ +// Copyright 2019-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_CRYPTO_OPENSSLCOMPAT_H +#define OES_CORE_CRYPTO_OPENSSLCOMPAT_H + +#include + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +/** + * In OpenSSL v1.1, some new functions were introduced, and other were renamed. + * This file allows the use of older OpenSSL versions by defining the new functions introduced by v1.1. + */ + +#include +#include +#include + + +int ECDSA_SIG_set0(ECDSA_SIG *sig, BIGNUM *r, BIGNUM *s) { + if (r == NULL || s == NULL) + return 0; + BN_clear_free(sig->r); + BN_clear_free(sig->s); + sig->r = r; + sig->s = s; + return 1; +} + +void ECDSA_SIG_get0(const ECDSA_SIG *sig, const BIGNUM **r, const BIGNUM **s) { + if (r != NULL) + *r = sig->r; + if (s != NULL) + *s = sig->s; +} + +EVP_MD_CTX *EVP_MD_CTX_new(void) { + return EVP_MD_CTX_create(); +} + +void EVP_MD_CTX_free(EVP_MD_CTX *ctx) { + EVP_MD_CTX_destroy(ctx); +} + +#endif + +#endif /* OES_CORE_CRYPTO_OPENSSLCOMPAT_H */ \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4ae0b6bd8b0784ef303b7e74656f8730b03f4d74 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/CMakeLists.txt @@ -0,0 +1,28 @@ +project(oes_http) +add_compile_definitions(oes_core_http_EXPORTS) + +# 3rd parties: + USE_CURL() + +# files needed for build + include(files.cmake) + +add_library( + oes_http SHARED + + ${oes_http_HEADERS} + ${oes_http_SOURCES} +) +target_link_libraries( + oes_http + # Internal library that are explicitelly inside header files + PUBLIC + + # Internal library that are only in source files + PRIVATE + + # 3rd Parties: + PRIVATE + $<$:pthread> + ${CURL_LIBRARIES} +) diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/HTTP.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/HTTP.cpp new file mode 100644 index 0000000000000000000000000000000000000000..529d9997c733f5aa44af1998956a4259e667d526 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/HTTP.cpp @@ -0,0 +1,353 @@ +// ============================================================================ +// Copyright 2018-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace oes { + namespace core { + static const char* s_ca_bundle = getenv("PG_HTTPS_CA_BUNDLE"); + + static std::set s_httpErrorCodes = { 408, 429, 500, 502, 503, 504 }; + + struct HTTPInitializer { + HTTPInitializer() { + curl_global_init(CURL_GLOBAL_DEFAULT); + } + + ~HTTPInitializer() { + curl_global_cleanup(); + } + }; + + struct HTTP::Impl { + enum class Method { + eGET, + ePOST, + ePUT, + eDELETE, + eHEAD + }; + + struct Request { + Request(); + ~Request(); + + bool run(Method method, const std::string& url, const size_t port, + const Header& inHeader, std::istream& inBody, long& outCode, + std::ostream& outHeader, std::ostream& outBody, const Options& opt); + + static size_t readCallback(void *target, size_t size, size_t nmemb, void *userData); + static size_t writeCallback(void *source, size_t size, size_t nmemb, void *userData); + + size_t streamlen(std::istream& inData); + + CURL* _curl = nullptr; + }; + + Response response(bool result, long code, std::stringstream& header, std::stringstream& body); + + Request _request; + #ifdef ASYNC + Request _asyncRequest; + DispatchQueue _dispatchQueue; + #endif + }; + + HTTP::Impl::Request::Request() { + /* Create a new curl handle */ + _curl = curl_easy_init(); + } + + HTTP::Impl::Request::~Request() { + if (_curl) { + /* always cleanup */ + curl_easy_cleanup(_curl); + } + } + + bool HTTP::Impl::Request::run(Method method, const std::string& url, const size_t port, + const Header& inHeader, std::istream& inBody, long& outCode, + std::ostream& outHeader, std::ostream& outBody, const Options& opt) { + + if (!_curl) { + outCode = CURLE_FAILED_INIT; + return false; + } + + unsigned int retry_count = 0; + + do { + if (retry_count > 0) { + auto seconds = std::min(600, (int)std::pow(2, retry_count - 1)); + std::this_thread::sleep_for(std::chrono::seconds(seconds)); + } + + /* First set the URL that is about to receive our command. */ + curl_easy_setopt(_curl, CURLOPT_URL, url.c_str()); + + /* Set the PORT. */ + if (port) { + curl_easy_setopt(_curl, CURLOPT_PORT, port); + } + + /* Get the custom headers */ + struct curl_slist* header = NULL; + if (inHeader.size() > 0) { + for (const auto& field : inHeader) { + header = curl_slist_append(header, field.c_str()); + } + } + + /* Set "Expect:" to disable "Expect: 100-continue" */ + header = curl_slist_append(header, "Expect:"); + + /* Set the custom headers */ + curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, header); + + if (method == Method::eGET) { + /* Now specify we want to GET data */ + curl_easy_setopt(_curl, CURLOPT_HTTPGET, 1L); + } else if (method == Method::ePOST) { + /* Now specify we want to POST data */ + curl_easy_setopt(_curl, CURLOPT_POST, 1L); + + /* We want to use our own read function */ + curl_easy_setopt(_curl, CURLOPT_READFUNCTION, readCallback); + + /* Pointer to pass to our read function */ + curl_easy_setopt(_curl, CURLOPT_READDATA, (void*)&inBody); + + /* Set the expected POST size. */ + curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)streamlen(inBody)); + } else if (method == Method::ePUT) { + /* Now specify we want to PUT data */ + curl_easy_setopt(_curl, CURLOPT_UPLOAD, 1L); + + /* We want to use our own read function */ + curl_easy_setopt(_curl, CURLOPT_READFUNCTION, readCallback); + + /* Pointer to pass to our read function */ + curl_easy_setopt(_curl, CURLOPT_READDATA, (void*)&inBody); + + /* Set the expected PUT size. */ + curl_easy_setopt(_curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)streamlen(inBody)); + } else if (method == Method::eDELETE) { + /* Now specify we want to DELETE data */ + curl_easy_setopt(_curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + } else if (method == Method::eHEAD) { + /* Now specify we want to HEAD data */ + curl_easy_setopt(_curl, CURLOPT_NOBODY, 1L); + } + + /* Send all data to this function */ + curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, writeCallback); + + /* We want the headers be written to this file handle */ + curl_easy_setopt(_curl, CURLOPT_HEADERDATA, (void*)&outHeader); + + /* We want the body be written to this stream instead of stdout */ + curl_easy_setopt(_curl, CURLOPT_WRITEDATA, (void*)&outBody); + + if (s_ca_bundle) { + /* Set path to ca bundle */ + curl_easy_setopt(_curl, CURLOPT_CAINFO, s_ca_bundle); + } else { + /* Disable peer verification */ + if (opt.disableVERIFYPEER) { + curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L); + } + if (opt.disableVERIFYHOST) { + curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L); + } + if (opt.disableVERIFYSTATUS) { + curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYSTATUS, 0L); + } + } + + if (opt.timeout > 0) { + curl_easy_setopt(_curl, CURLOPT_TIMEOUT, opt.timeout); + } + + /* Get verbose debug output please */ + //curl_easy_setopt(_curl, CURLOPT_VERBOSE, 1L); + + /* Perform the request, res will get the return code */ + CURLcode res = curl_easy_perform(_curl); + /* Check for errors */ + if (res != CURLE_OK) { + outCode = res; + + /* Always reset */ + curl_easy_reset(_curl); + + if (header) { + /* Free the custom headers */ + curl_slist_free_all(header); + } + + return false; + } + + curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &outCode); + + /* Always reset */ + curl_easy_reset(_curl); + + if (header) { + /* Free the custom headers */ + curl_slist_free_all(header); + } + + retry_count++; + + } while (retry_count < opt.retry && s_httpErrorCodes.find(outCode) != s_httpErrorCodes.end()); + + return true; + } + + size_t HTTP::Impl::Request::readCallback( + void *target, size_t size, size_t nmemb, void *userData) { + auto req_body = (std::istream*)userData; + + try { + req_body->read((char*)target, size * nmemb); + } + catch (...) { + return 0; + } + + return req_body->gcount(); + } + + size_t HTTP::Impl::Request::writeCallback( + void *source, size_t size, size_t nmemb, void *userData) { + auto res_body = (std::ostream*)userData; + + try { + res_body->write((const char*)source, size * nmemb); + } + catch (...) { + return 0; + } + + return size * nmemb; + } + + size_t HTTP::Impl::Request::streamlen(std::istream& stream) { + stream.seekg(0, std::ios_base::end); + size_t length = stream.tellg(); + stream.seekg(0); + + return length; + } + + HTTP::Response HTTP::Impl::response(bool result, + long code, std::stringstream& header, std::stringstream& body) { + Response res; + + if (result) { + res.code = (short)code; + std::getline(header, res.status, '\r'); + header.seekg(1, std::ios_base::cur); // skip '\n' + + for (std::string field; std::getline(header, field, '\r'); ) { + if (!field.empty()) { + auto key_end = field.find_first_of(": "); + res.fields.emplace( + field.substr(0, key_end), + field.substr(key_end + 2, field.length() - key_end)); + } + header.seekg(1, std::ios_base::cur); // skip '\n' + } + + res.body = body.str(); + } + else { + res.code = -1; + res.status = curl_easy_strerror((CURLcode)code); + } + + return res; + } + + HTTP::HTTP() : _impl(new Impl()) { + static HTTPInitializer init; + } + + HTTP::~HTTP() { + delete _impl; + } + + HTTP::Response HTTP::get(const std::string& url, const size_t port, + const Header& header, const Options& opt) { + std::stringstream req_body, res_header, res_body; + long res_code; + bool res = _impl->_request.run(Impl::Method::eGET, url, port, + header, req_body, res_code, res_header, res_body, opt); + + return _impl->response(res, res_code, res_header, res_body); + } + + HTTP::Response HTTP::post(const std::string& url, const size_t port, + const Header& header, const std::string& body, const Options& opt) { + std::stringstream req_body(body), res_header, res_body; + long res_code; + bool res = _impl->_request.run( + Impl::Method::ePOST, url, port, header, req_body, res_code, res_header, res_body, opt); + + return _impl->response(res, res_code, res_header, res_body); + } + + HTTP::Response HTTP::put(const std::string& url, const size_t port, + const Header& header, const std::string& body, const Options& opt) { + std::stringstream req_body(body), res_header, res_body; + long res_code; + bool res = _impl->_request.run( + Impl::Method::ePUT, url, port, header, req_body, res_code, res_header, res_body, opt); + + return _impl->response(res, res_code, res_header, res_body); + } + + + HTTP::Response HTTP::del(const std::string& url, const size_t port, const Header& header, const Options& opt) { + std::stringstream req_body, res_header, res_body; + long res_code; + bool res = _impl->_request.run( + Impl::Method::eDELETE, url, port, header, req_body, res_code, res_header, res_body, opt); + + return _impl->response(res, res_code, res_header, res_body); + } + + HTTP::Response HTTP::head(const std::string& url, const size_t port, const Options& opt) { + std::stringstream req_body, res_header, res_body; + long res_code; + bool res = _impl->_request.run( + Impl::Method::eHEAD, url, port, {}, req_body, res_code, res_header, res_body, opt); + + return _impl->response(res, res_code, res_header, res_body); + } + + } // namespace core +} // namespace oes \ No newline at end of file diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/HTTP.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/HTTP.h new file mode 100644 index 0000000000000000000000000000000000000000..6637ab51a8075297724fc6b791dc72047e101fa8 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/HTTP.h @@ -0,0 +1,182 @@ +// ============================================================================ +// Copyright 2018-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_HTTP_HTTP_H +#define OES_CORE_HTTP_HTTP_H + +/*! \file HTTP.h +* \brief Describes a \c HTTP class +*/ + +#include + +#include +#include +#include +#include + +namespace oes { + namespace core { + + /*! \class HTTP + * \brief Simple HTTP client implementation + * \ingroup oes_core_http + * \details + * HTTP class provides HTTP sync and async methods: + * GET, POST, PUT, GELETE and HEAD + * + * \code + * HTTP http; + * HTTP::Response response; + * HTTP::AsyncResponse async_response; + * + * response = http.get("https://example.com/", 0, { "Accept: text/html" }); + * + * async_response = http.async_post("https://example.com/", 0, { "Content-Type: application/json" }, R"({ "JSON": "struct"})"); + * response = async_response.get(); + * \endcode + * + * Note: By default HTTPS certificate verification is disabled. + * To enable HTTPS certificate verification, set PG_HTTPS_CA_BUNDLE to the CA bundle file. + * The latest CA bundle file can be found here https://curl.haxx.se/docs/caextract.html + */ + class OES_CORE_HTTP_EXPORT HTTP + { + public: + using Header = std::vector; + + using Fields = std::map; + + /*! \struct Response + * \brief HTTP response + * + * Note: If the http method was successful, code/status contains + * the server's response code/status. + * Otherwise, in case of internal failure, the code is -1 + * and the status contains an internal error message. + * + * \ingroup oes_core_http + */ + struct Response { + Response() {} + short code = 0; + std::string status; + Fields fields; + std::string body; + }; + + using AsyncResponse = std::future; + + /*! \struct Options + * \brief HTTP request options + * + * \ingroup oes_core_http + */ + struct Options { + Options() {} + /* request timeout, 0 by default */ + long timeout = 0; + /* true to disable CURLOPT_SSL_VERIFYPEER when there is no certificate bundle, true by default*/ + bool disableVERIFYPEER = true; + /* true to disable CURLOPT_SSL_VERIFYHOST when there is no certificate bundle, false by default*/ + bool disableVERIFYHOST = false; + /* true to disable CURLOPT_SSL_VERIFYSTATUS when there is no certificate bundle, false by default*/ + bool disableVERIFYSTATUS = false; + /* number of retries on request error, exponential delay starting at 1 second, 0 by default */ + unsigned int retry = 0; + }; + + HTTP(); + virtual ~HTTP(); + + HTTP(const HTTP&) = delete; + HTTP& operator=(const HTTP&) = delete; + + /*! \brief HTTP GET method + * + * \param url HTTP server URL + * \param port HTTP server port + * \param header HTTP header fields + * \param opt HTTP request options + * + * \retval Response Server's response. + * + * \ingroup oes_core_http + */ + virtual Response get(const std::string& url, const size_t port, const Header& header, const Options& opt = Options()); + + /*! \brief HTTP POST method + * + * \param url HTTP server URL + * \param port HTTP server port + * \param header HTTP header fields + * \param body HTTP body + * \param opt HTTP request options + * + * \retval Response Server's response. + * + * \ingroup oes_core_http + */ + virtual Response post(const std::string& url, const size_t port, const Header& header, const std::string& body, const Options& opt = Options()); + + /*! \brief HTTP PUT method + * + * \param url HTTP server URL + * \param port HTTP server port + * \param header HTTP header fields + * \param body HTTP body + * \param opt HTTP request options + * + * \retval Response Server's response. + * + * \ingroup oes_core_http + */ + virtual Response put(const std::string& url, const size_t port, const Header& header, const std::string& body, const Options& opt = Options()); + + /*! \brief HTTP DELETE method + * + * \param url HTTP server URL + * \param port HTTP server port + * \param header HTTP header fields + * \param opt HTTP request options + * + * \retval Response Server's response. + * + * \ingroup oes_core_http + */ + virtual Response del(const std::string& url, const size_t port, const Header& header, const Options& opt = Options()); + + /*! \brief HTTP HEAD method + * + * \param url HTTP server URL + * \param port HTTP server port + * \param opt HTTP request options + * + * \retval Response Server's response. + * + * \ingroup oes_core_http + */ + virtual Response head(const std::string& url, const size_t port, const Options& opt = Options()); + + private: + struct Impl; + Impl* _impl; + }; + + } // namespace core +} // namespace oes + +#endif /* OES_CORE_HTTP_HTTP_H */ diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/export.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/export.h new file mode 100644 index 0000000000000000000000000000000000000000..db4c49aa708cafc99ece05e594fa59d6934880b8 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/export.h @@ -0,0 +1,33 @@ +// ============================================================================ +// Copyright 2018-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_HTTP_EXPORT_H +#define OES_CORE_HTTP_EXPORT_H + +#if ! defined(OES_CORE_HTTP_EXPORT) +# if defined(_WIN32) +# if defined(oes_core_http_EXPORTS) +# define OES_CORE_HTTP_EXPORT __declspec(dllexport) +# else +# define OES_CORE_HTTP_EXPORT __declspec(dllimport) +# endif +# else +# define OES_CORE_HTTP_EXPORT +# endif +#endif + +#endif + diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/files.cmake b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/files.cmake new file mode 100644 index 0000000000000000000000000000000000000000..0ffc61f392cfbfedcd7a8a16bbdf040d64562bee --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/http/files.cmake @@ -0,0 +1,9 @@ +################################################################################ +# http directory +set( oes_http_SOURCES + HTTP.cpp +) +set( oes_http_HEADERS + HTTP.h + export.h +) diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Assert.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Assert.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fe8fbc463b34c08335e2e9733a42070c800df531 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Assert.cpp @@ -0,0 +1,327 @@ +// ============================================================================ +// Copyright 2010-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include + +#include +#include "fmt/format.h" + +#include +#include + +#if defined(_WIN32) + #define VC_EXTRALEAN + #define WIN32_LEAN_AND_MEAN + #include +#else + #include +#endif +#include + +namespace oes { + namespace core { + + namespace { + void defaultAssertionHandler(const AssertionInfo& info) { + // Default assertion policy is to: + // - Send a fatal error message + // - abort in debug build + std::cerr << fmt::format( + "{} ({}). \nIn file: {}\nIn function: {}\nAt line: {}", + info.message, info.condition, info.file, info.function, info.line + ) << std::endl; + +#if defined (OES_DEBUG_BUILD) + std::abort(); +#endif + } + + AssertionHandler& getAssertionHandlerRef() { + static AssertionHandler handler = defaultAssertionHandler; + return handler; + } + + bool defaultBreakpointHandler() { + // Default breakpoint policy is to: + // - Never break in release builds + // - Always break in Windows debug builds + // - Conditionally break in Linux debug builds +#if defined (OES_DEBUG_BUILD) + #if defined(_WIN32) + return true; + #else + return detail::isDebuggerAttached(); + #endif +#else + return false; +#endif + } + + BreakpointHandler& getBreakpointHandlerRef() { + static BreakpointHandler handler = defaultBreakpointHandler; + return handler; + } + } + + /*! \struct AssertionInfo + * \brief Describes the context of an assertion/check failure + * \ingroup oes_core_utils_assertion + */ + + /*! \enum AssertionInfo::Type + * \brief Type of assertion failure (check/assert) + * + * \var AssertionInfo::Type AssertionInfo::AssertFailed + * When using OES_ASSERT + * + * \var AssertionInfo::Type AssertionInfo::DebugAssertFailed + * When using OES_DEBUG_ASSERT + */ + + /*! \var AssertionInfo::Type AssertionInfo::type + * \brief Type of assertion failure (check/require/assert) + */ + + /*! \var String AssertionInfo::message + * \brief A message (if any) associated with the assertion + */ + + /*! \var String AssertionInfo::condition + * \brief The condition (in its textual form) that triggered the assertion failure + */ + + /*! \var String AssertionInfo::file + * \brief The file where the assertion failed (as provided by compiler intrinsic __FILE__) + */ + + /*! \var String AssertionInfo::function + * \brief The function where the assertion failed (as provided by compiler intrinsic __FUNCTION__) + */ + + /*! \var int AssertionInfo::line + * \brief The line where the assertion failed (as provided by compiler intrinsic __LINE__) + */ + + /*! \typedef boost::function AssertionHandler + * \brief Type of the function object used to handle assertion failure + * \ingroup oes_core_utils_assertion + * \see getAssertionHandler setAssertionHandler + */ + + /*! \brief Retrieves the current assertion handler + * \ingroup oes_core_utils_assertion + * \details + * The assertion handler is invoked whenever an assertion failed, with relevant + * argument describing current assertion failure context. Most typically, it is + * automatically invoked as part of OES_ASSERT and OES_EXPENSIVE_ASSERT + * macros. + * \return Current assertion handler (or a a null function object if not set) + */ + const AssertionHandler getAssertionHandler() { + return getAssertionHandlerRef(); + } + + /*! \brief Sets a new assertion handler + * \ingroup oes_core_utils_assertion + * \param[in] handler A new assertion handler to use (may be a null function object) + */ + void setAssertionHandler(const AssertionHandler& handler) { + getAssertionHandlerRef() = handler; + } + + /*! \typedef boost::function BreakpointHandler + * \brief Type of the function object used to handle breakpoint invocation + * \ingroup oes_core_utils_assertion + * \details + * A breakpoint handler should return \c true if client may proceed and + * trigger a breakpoint, \c false otherwise (breakpoint disabled). + * \see getBreakpointHandler setBreakpointHandler + */ + + /*! \brief Retrieves the current breakpoint handler + * \ingroup oes_core_utils_assertion + * \details + * The breakpoint handler is invoked right before a breakpoint is triggered + * programmatically using OES_BREAK_INTERNAL() macro. + * \return Current breakpoint handler (or a default handler if not set, see note) + * \note A default handler is created and returned when this function is invoked + * for the first time. The default handler works as follows: + * - In all release builds, returns false (i.e. do not break) + * - In Windows debug builds, returns true (i.e. do break) + * - In Linux debug builds, returns true if a debugger is attached (i.e. do break), + * false otherwise (i.e. do not break). + */ + const BreakpointHandler getBreakpointHandler() { + return getBreakpointHandlerRef(); + } + + /*! \brief Sets a new breakpoint handler + * \ingroup oes_core_utils_assertion + * \param[in] handler A new breakpoint handler to use (may be a null function object) + */ + void setBreakpointHandler(const BreakpointHandler& handler) { + getBreakpointHandlerRef() = handler; + } + + namespace detail { + bool shouldBreak() { + const BreakpointHandler handler = getBreakpointHandler(); + return handler && handler(); + } + + void handleAssertionFailure( + AssertionInfo::Type type, + const char* condition, + const char* message, + const char* file, + const char* function, + int line) { + const AssertionHandler handler = getAssertionHandler(); + if( handler ) { + const AssertionInfo info( + type, + condition ? condition : String(), + message ? message : String(), + file ? file : String(), + function ? function : String(), + line); + handler(info); + } + } + + bool isDebuggerAttached() { +#if defined(_WIN32) + return (IsDebuggerPresent()!=0); +#else + // From Topology's assert. This call to ptrace will return an + // error if the current process is being traced (running with debugger + // or debugger attached) and prevent program to crash if a breakpoint + // is triggered while not under debugger. + // Notorious side-effects: the ptrace function can only be called ONCE, + // debugger cannot be attached afterwards. + static const bool value = (ptrace(PTRACE_TRACEME, 0, nullptr, nullptr) < 0); + return value; +#endif + } + } // End namespace detail + + } // End namespace core +} // End namespace oes + +/*! \defgroup oes_core_utils_assertion Assertion handling and macros + * \ingroup oes_core_utils + * \warning Read the following carefully as OES assertion behavior may differ + * from other code bases. + * + * \par Macros + * Two macros are available: + * \li OES_ASSERT should be used whenever possible to check and enforce + * invariants, pre-conditions and post-conditions. This type of assertion + * is ALWAYS enabled and include a fallback action. + * \li OES_DEBUG_ASSERT should be used when it is potentially computationally-expensive + * to evaluate the condition or should be enforced in debug mode only. This type of + * assertion is enabled in debug mode ONLY and does not include a fallback action. + * + * \par Macros and compilation mode + * Not all macros are created equal, beware of side-effects in the evaluation of + * the condition. + * \li OES_ASSERT is ALWAYS enabled, the condition is always evaluated regardless of + * compilation mode (debug/release/...) + * \li OES_DEBUG_ASSERT is ONLY enabled in debug mode, the condition is therefore + * NOT evaluated in any other mode. + * + * \par Behavior in DEBUG mode: + * \li The behavior when an assertion fails is to check if breakpoints are enabled, + * breaking if needed, to call the assertion handler and finally to ABORT (REGARDLESS + * of any fallback action). + * \li The default breakpoint policy is to always break in Windows debug builds and break + * only if a debugger is attached in Linux debug builds. + * + * \par Behavior in RELEASE mode: + * \li The behavior when an assertion fails is to call the assertion handler and execute + * the fallback action provided as input. + * + * \par Assertion and breakpoint handlers + * The default behavior can be changed as follows: + * \li setBreakpointHandler() can be used to provide a new breakpoint handling strategy. + * The breakpoint is triggered only if the breakpoint handler exists and returns \c true. + * \li setAssertionHandler() can be used to provide a new assertion handling strategy. + * The assertion handler is called whenever the expression evaluated by OES_ASSERT + * or OES_DEBUG_ASSERT() fails. + */ + +/* \def OES_ASSERT(condition,action) + * \brief Macro used to check that a condition is met + * \ingroup oes_core_utils_assertion + * \details + * \li If breakpoints are enabled (i.e. the breakpoint handler exists and returns + * \c true, \see getBreakpointHandler), a breakpoint is triggered before proceeding. + * \li If an assertion handler exists (\see getAssertionHandler), an assertion context is + * created (\see AssertionInfo) and the assertion handler is invoked. + * \li In DEBUG builds, execution flow is stops here and the program aborts. In RELEASE + * builds, the execution continues starting with the fallback action provided as input. + * + * \note This type of assertion is always enabled (in release mode as well as any other + * mode). \p condition is always evaluated. + */ + +/* \def OES_DEBUG_ASSERT(condition) + * \brief Macro used to enforce (in DEBUG mode only) that a required condition is met + * \ingroup oes_core_utils_assertion + * \details + * \li If breakpoints are enabled (meaning that the breakpoint handler returns \c true, + * \see getBreakpointHandler), a breakpoint is triggered before proceeding. + * \li If an assertion handler exists (\see getAssertionHandler), an assertion context is + * created (\see AssertionInfo) and the handler is invoked. + * \li The execution flow stops here and the program aborts. + * + * \warning This type of assertion is only enabled in debug mode, \p condition + * is NOT evaluated in non-debug mode. + */ + +/* \def OES_ASSERT_MSG(condition, oss_args, action) +* \brief Macro used to check that a condition is met, and send a message when it is not the case. +* \ingroup oes_core_utils_assertion +* \details +* \li If breakpoints are enabled (i.e. the breakpoint handler exists and returns +* \c true, \see getBreakpointHandler), a breakpoint is triggered before proceeding. +* \li If an assertion handler exists (\see getAssertionHandler), an assertion context is +* created (\see AssertionInfo) and the assertion handler is invoked. +* \li In DEBUG builds, execution flow stops here and the program aborts. In RELEASE +* builds, the execution continues starting with the fallback action provided as input. +* +* \note This type of assertion is always enabled (in release mode as well as any other +* mode). \p condition is always evaluated. +*/ + +/*! \internal + * \def OES_DO_NOTHING_STATEMENT + * \brief DO NOT USE - Implementation macro defining the "do nothing" statement + * \ingroup oes_core_utils_assertion + * \details + * This is required to implement correctly most macros and prevent un-wanted expansion + * when macro is empty or nested within control statements (e.g. if). + * Forces client to terminate macro calls with a semi-colon + */ + +/*! \internal + * \def OES_BREAK_INTERNAL + * \brief DO NOT USE - Implementation macro defining a breakpoint invocation + * \ingroup oes_core_utils_assertion + * \par Supported platforms and OS + * \li Windows using Microsoft (cl) or Intel (icc) compilers + * \li Linux using GCC on Intel (g++) or Intel (icc) compilers + */ diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Assert.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Assert.h new file mode 100644 index 0000000000000000000000000000000000000000..1ecaa381f7836f0aefdda58b5610790cc28d987e --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Assert.h @@ -0,0 +1,154 @@ +// ============================================================================ +// Copyright 2010-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_ASSERT_H +#define OES_CORE_UTILS_ASSERT_H + +#include +#include + +#include +#include +#include +#include + +namespace oes { + namespace core { + + struct OES_CORE_UTILS_EXPORT AssertionInfo { + enum Type { Unknown, AssertFailed, DebugAssertFailed }; + Type type; + String message; + String condition; + String file; + String function; + int line; + + AssertionInfo( + Type t, const String& m, const String& c, const String& f, const String& fn, int l) : + type(t), message(m), condition(c), file(f), function(fn), line(l) { + } + AssertionInfo(): + type(Unknown), line(-1) { + } + }; + + typedef boost::function BreakpointHandler; + OES_CORE_UTILS_EXPORT const BreakpointHandler getBreakpointHandler(); + OES_CORE_UTILS_EXPORT void setBreakpointHandler(const BreakpointHandler& handler); + + typedef boost::function AssertionHandler; + OES_CORE_UTILS_EXPORT const AssertionHandler getAssertionHandler(); + OES_CORE_UTILS_EXPORT void setAssertionHandler(const AssertionHandler& handler); + + namespace detail { + // To avoid compilation warning(s) when evaluating assertion + inline bool is_true(bool condition) { + return condition; + } + + OES_CORE_UTILS_EXPORT bool isDebuggerAttached(); + OES_CORE_UTILS_EXPORT bool shouldBreak(); + OES_CORE_UTILS_EXPORT void handleAssertionFailure( + AssertionInfo::Type type, + const char* condition, + const char* message, + const char* file, + const char* function, + int line); + } // End namespace detail + + } // End namespace core +} // End namespace oes + +#define OES_DO_NOTHING_STATEMENT ((void)0) + + +// Definition of OES_BREAK_INTERNAL + +// MSVC or Intel C++ compilers +#if defined(_MSC_VER) || defined( __INTEL_COMPILER) + #define OES_BREAK_INTERNAL()\ + __debugbreak() +// GNU GCC on Intel 32 & 64 platforms +#elif defined( __GNUC__ ) && ( defined(__i386__) || defined(__x86_64__) ) + #define OES_BREAK_INTERNAL()\ + __asm__ __volatile__ ( "int $3" ) +// Unknown platforms or compilers +#else + #define OES_BREAK_INTERNAL()\ + OES_DO_NOTHING_STATEMENT +#endif + +// Definition of OES_ASSERT_MSG + +#define OES_ASSERT_MSG(condition,oss_args,action)\ + if (not oes::core::detail::is_true(bool(condition)) ) {\ + if (oes::core::detail::shouldBreak()) {\ + OES_BREAK_INTERNAL();\ + }\ + std::ostringstream oss;\ + oss << "Assertion failed: " << oss_args;\ + const std::string msg = oss.str();\ + oes::core::detail::handleAssertionFailure(\ + oes::core::AssertionInfo::AssertFailed, msg.c_str(),\ + #condition,\ + __FILE__,\ + BOOST_CURRENT_FUNCTION,\ + __LINE__\ + );\ + action\ + } else OES_DO_NOTHING_STATEMENT + + +// Definition of OES_ASSERT_INTERNAL + +#define OES_ASSERT_INTERNAL(condition,type,msg,action)\ + if( not oes::core::detail::is_true(bool(condition)) ) {\ + if( oes::core::detail::shouldBreak() ) {\ + OES_BREAK_INTERNAL();\ + }\ + oes::core::detail::handleAssertionFailure(\ + type,\ + msg,\ + #condition,\ + __FILE__,\ + BOOST_CURRENT_FUNCTION,\ + __LINE__\ + );\ + action\ + } else OES_DO_NOTHING_STATEMENT + +// Definition of OES_ASSERT + +#define OES_ASSERT(condition,action)\ + OES_ASSERT_INTERNAL(condition, oes::core::AssertionInfo::AssertFailed,\ + "Assertion failed",\ + action) + +// Definition of OES_DEBUG_ASSERT + +#if defined (OES_DEBUG_BUILD) + #define OES_DEBUG_ASSERT(condition)\ + OES_ASSERT_INTERNAL(condition, oes::core::AssertionInfo::DebugAssertFailed,\ + "Debug assertion failed",\ + OES_DO_NOTHING_STATEMENT;) +#else + #define OES_DEBUG_ASSERT(condition)\ + OES_DO_NOTHING_STATEMENT +#endif + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/CMakeLists.txt b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b0bbbd6f7ff51a06d225b7b0f2e5cc91f14bd95d --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/CMakeLists.txt @@ -0,0 +1,29 @@ +project(oes_utils) +add_compile_definitions(oes_core_utils_EXPORTS) + +# 3rd parties: + USE_BOOST() + +# files needed for build + include(files.cmake) + +add_library( + oes_utils SHARED + + ${oes_utils_HEADERS} + ${oes_utils_SOURCES} +) +target_link_libraries( + oes_utils + # Internal library that are explicitelly inside header files + PUBLIC + + # Internal library that are only in source files + PRIVATE + + # 3rd Parties: + PRIVATE + $<$:Ws2_32.lib> + ${BOOST_LIBRARIES} + ${OPENSSL_LIBRARIES} +) diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoDate.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoDate.h new file mode 100644 index 0000000000000000000000000000000000000000..8589b120b7bba1bde621cfd6ca244a7ae1b29345 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoDate.h @@ -0,0 +1,8008 @@ +#ifndef DATE_H +#define DATE_H + +// The MIT License (MIT) +// +// Copyright (c) 2015, 2016, 2017 Howard Hinnant +// Copyright (c) 2016 Adrian Colomitchi +// Copyright (c) 2017 Florian Dang +// Copyright (c) 2017 Paul Thompson +// Copyright (c) 2018, 2019 Tomasz Kamiński +// Copyright (c) 2019 Jiangang Zhuang +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Our apologies. When the previous paragraph was written, lowercase had not yet +// been invented (that would involve another several millennia of evolution). +// We did not mean to shout. + +#ifndef HAS_STRING_VIEW +# if __cplusplus >= 201703 || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +# define HAS_STRING_VIEW 1 +# else +# define HAS_STRING_VIEW 0 +# endif +#endif // HAS_STRING_VIEW + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HAS_STRING_VIEW +# include +#endif +#include +#include + +#ifdef __GNUC__ +# pragma GCC diagnostic push +# if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 7) +# pragma GCC diagnostic ignored "-Wpedantic" +# endif +# if __GNUC__ < 5 +// GCC 4.9 Bug 61489 Wrong warning with -Wmissing-field-initializers +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" +# endif +#endif + +#ifdef _MSC_VER +# pragma warning(push) +// warning C4127: conditional expression is constant +# pragma warning(disable : 4127) +#endif + +namespace date +{ + +//---------------+ +// Configuration | +//---------------+ + +#ifndef ONLY_C_LOCALE +# define ONLY_C_LOCALE 0 +#endif + +#if defined(_MSC_VER) && (!defined(__clang__) || (_MSC_VER < 1910)) +// MSVC +# ifndef _SILENCE_CXX17_UNCAUGHT_EXCEPTION_DEPRECATION_WARNING +# define _SILENCE_CXX17_UNCAUGHT_EXCEPTION_DEPRECATION_WARNING +# endif +# if _MSC_VER < 1910 +// before VS2017 +# define CONSTDATA const +# define CONSTCD11 +# define CONSTCD14 +# define NOEXCEPT _NOEXCEPT +# else +// VS2017 and later +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 constexpr +# define NOEXCEPT noexcept +# endif + +#elif defined(__SUNPRO_CC) && __SUNPRO_CC <= 0x5150 +// Oracle Developer Studio 12.6 and earlier +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 +# define NOEXCEPT noexcept + +#elif __cplusplus >= 201402 +// C++14 +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 constexpr +# define NOEXCEPT noexcept +#else +// C++11 +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 +# define NOEXCEPT noexcept +#endif + +#ifndef HAS_UNCAUGHT_EXCEPTIONS +# if __cplusplus > 201703 || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L) +# define HAS_UNCAUGHT_EXCEPTIONS 1 +# else +# define HAS_UNCAUGHT_EXCEPTIONS 0 +# endif +#endif // HAS_UNCAUGHT_EXCEPTIONS + +#ifndef HAS_VOID_T +# if __cplusplus >= 201703 || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +# define HAS_VOID_T 1 +# else +# define HAS_VOID_T 0 +# endif +#endif // HAS_VOID_T + +// Protect from Oracle sun macro +#ifdef sun +# undef sun +#endif + +// Work around for a NVCC compiler bug which causes it to fail +// to compile std::ratio_{multiply,divide} when used directly +// in the std::chrono::duration template instantiations below +namespace detail { +template +using ratio_multiply = decltype(std::ratio_multiply{}); + +template +using ratio_divide = decltype(std::ratio_divide{}); +} // namespace detail + + //-----------+ + // Interface | + //-----------+ + + // durations + +using days = std::chrono::duration +, std::chrono::hours::period>>; + +using weeks = std::chrono::duration +, days::period>>; + +using years = std::chrono::duration +, days::period>>; + +using months = std::chrono::duration +>>; + +// time_point + +template +using sys_time = std::chrono::time_point; + +using sys_days = sys_time; +using sys_seconds = sys_time; + +struct local_t {}; + +template +using local_time = std::chrono::time_point; + +using local_seconds = local_time; +using local_days = local_time; + +// types + +struct last_spec +{ + explicit last_spec() = default; +}; + +class day; +class month; +class year; + +class weekday; +class weekday_indexed; +class weekday_last; + +class month_day; +class month_day_last; +class month_weekday; +class month_weekday_last; + +class year_month; + +class year_month_day; +class year_month_day_last; +class year_month_weekday; +class year_month_weekday_last; + +// date composition operators + +CONSTCD11 year_month operator/(const year& y, const month& m) NOEXCEPT; +CONSTCD11 year_month operator/(const year& y, int m) NOEXCEPT; + +CONSTCD11 month_day operator/(const day& d, const month& m) NOEXCEPT; +CONSTCD11 month_day operator/(const day& d, int m) NOEXCEPT; +CONSTCD11 month_day operator/(const month& m, const day& d) NOEXCEPT; +CONSTCD11 month_day operator/(const month& m, int d) NOEXCEPT; +CONSTCD11 month_day operator/(int m, const day& d) NOEXCEPT; + +CONSTCD11 month_day_last operator/(const month& m, last_spec) NOEXCEPT; +CONSTCD11 month_day_last operator/(int m, last_spec) NOEXCEPT; +CONSTCD11 month_day_last operator/(last_spec, const month& m) NOEXCEPT; +CONSTCD11 month_day_last operator/(last_spec, int m) NOEXCEPT; + +CONSTCD11 month_weekday operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT; +CONSTCD11 month_weekday operator/(int m, const weekday_indexed& wdi) NOEXCEPT; +CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT; +CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, int m) NOEXCEPT; + +CONSTCD11 month_weekday_last operator/(const month& m, const weekday_last& wdl) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(int m, const weekday_last& wdl) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, const month& m) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, int m) NOEXCEPT; + +CONSTCD11 year_month_day operator/(const year_month& ym, const day& d) NOEXCEPT; +CONSTCD11 year_month_day operator/(const year_month& ym, int d) NOEXCEPT; +CONSTCD11 year_month_day operator/(const year& y, const month_day& md) NOEXCEPT; +CONSTCD11 year_month_day operator/(int y, const month_day& md) NOEXCEPT; +CONSTCD11 year_month_day operator/(const month_day& md, const year& y) NOEXCEPT; +CONSTCD11 year_month_day operator/(const month_day& md, int y) NOEXCEPT; + +CONSTCD11 +year_month_day_last operator/(const year_month& ym, last_spec) NOEXCEPT; +CONSTCD11 +year_month_day_last operator/(const year& y, const month_day_last& mdl) NOEXCEPT; +CONSTCD11 +year_month_day_last operator/(int y, const month_day_last& mdl) NOEXCEPT; +CONSTCD11 +year_month_day_last operator/(const month_day_last& mdl, const year& y) NOEXCEPT; +CONSTCD11 +year_month_day_last operator/(const month_day_last& mdl, int y) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const year& y, const month_weekday& mwd) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(int y, const month_weekday& mwd) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const month_weekday& mwd, const year& y) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const month_weekday& mwd, int y) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(int y, const month_weekday_last& mwdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const month_weekday_last& mwdl, int y) NOEXCEPT; + +// Detailed interface + +// day + +class day +{ + unsigned char d_; + +public: + day() = default; + explicit CONSTCD11 day(unsigned d) NOEXCEPT; + + CONSTCD14 day& operator++() NOEXCEPT; + CONSTCD14 day operator++(int) NOEXCEPT; + CONSTCD14 day& operator--() NOEXCEPT; + CONSTCD14 day operator--(int) NOEXCEPT; + + CONSTCD14 day& operator+=(const days& d) NOEXCEPT; + CONSTCD14 day& operator-=(const days& d) NOEXCEPT; + + CONSTCD11 explicit operator unsigned() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator!=(const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator< (const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator> (const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator<=(const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator>=(const day& x, const day& y) NOEXCEPT; + +CONSTCD11 day operator+(const day& x, const days& y) NOEXCEPT; +CONSTCD11 day operator+(const days& x, const day& y) NOEXCEPT; +CONSTCD11 day operator-(const day& x, const days& y) NOEXCEPT; +CONSTCD11 days operator-(const day& x, const day& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const day& d); + +// month + +class month +{ + unsigned char m_; + +public: + month() = default; + explicit CONSTCD11 month(unsigned m) NOEXCEPT; + + CONSTCD14 month& operator++() NOEXCEPT; + CONSTCD14 month operator++(int) NOEXCEPT; + CONSTCD14 month& operator--() NOEXCEPT; + CONSTCD14 month operator--(int) NOEXCEPT; + + CONSTCD14 month& operator+=(const months& m) NOEXCEPT; + CONSTCD14 month& operator-=(const months& m) NOEXCEPT; + + CONSTCD11 explicit operator unsigned() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator< (const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator> (const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator<=(const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator>=(const month& x, const month& y) NOEXCEPT; + +CONSTCD14 month operator+(const month& x, const months& y) NOEXCEPT; +CONSTCD14 month operator+(const months& x, const month& y) NOEXCEPT; +CONSTCD14 month operator-(const month& x, const months& y) NOEXCEPT; +CONSTCD14 months operator-(const month& x, const month& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month& m); + +// year + +class year +{ + short y_; + +public: + year() = default; + explicit CONSTCD11 year(int y) NOEXCEPT; + + CONSTCD14 year& operator++() NOEXCEPT; + CONSTCD14 year operator++(int) NOEXCEPT; + CONSTCD14 year& operator--() NOEXCEPT; + CONSTCD14 year operator--(int) NOEXCEPT; + + CONSTCD14 year& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 year operator-() const NOEXCEPT; + CONSTCD11 year operator+() const NOEXCEPT; + + CONSTCD11 bool is_leap() const NOEXCEPT; + + CONSTCD11 explicit operator int() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; + + static CONSTCD11 year min() NOEXCEPT { return year{-32767}; } + static CONSTCD11 year max() NOEXCEPT { return year{32767}; } +}; + +CONSTCD11 bool operator==(const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator!=(const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator< (const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator> (const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator<=(const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator>=(const year& x, const year& y) NOEXCEPT; + +CONSTCD11 year operator+(const year& x, const years& y) NOEXCEPT; +CONSTCD11 year operator+(const years& x, const year& y) NOEXCEPT; +CONSTCD11 year operator-(const year& x, const years& y) NOEXCEPT; +CONSTCD11 years operator-(const year& x, const year& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year& y); + +// weekday + +class weekday +{ + unsigned char wd_; +public: + weekday() = default; + explicit CONSTCD11 weekday(unsigned wd) NOEXCEPT; + CONSTCD14 weekday(const sys_days& dp) NOEXCEPT; + CONSTCD14 explicit weekday(const local_days& dp) NOEXCEPT; + + CONSTCD14 weekday& operator++() NOEXCEPT; + CONSTCD14 weekday operator++(int) NOEXCEPT; + CONSTCD14 weekday& operator--() NOEXCEPT; + CONSTCD14 weekday operator--(int) NOEXCEPT; + + CONSTCD14 weekday& operator+=(const days& d) NOEXCEPT; + CONSTCD14 weekday& operator-=(const days& d) NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; + + CONSTCD11 unsigned c_encoding() const NOEXCEPT; + CONSTCD11 unsigned iso_encoding() const NOEXCEPT; + + CONSTCD11 weekday_indexed operator[](unsigned index) const NOEXCEPT; + CONSTCD11 weekday_last operator[](last_spec) const NOEXCEPT; + +private: + static CONSTCD14 unsigned char weekday_from_days(int z) NOEXCEPT; + + friend CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; + friend CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; + friend CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; + template + friend std::basic_ostream& + operator<<(std::basic_ostream& os, const weekday& wd); + friend class weekday_indexed; +}; + +CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; +CONSTCD11 bool operator!=(const weekday& x, const weekday& y) NOEXCEPT; + +CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; +CONSTCD14 weekday operator+(const days& x, const weekday& y) NOEXCEPT; +CONSTCD14 weekday operator-(const weekday& x, const days& y) NOEXCEPT; +CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday& wd); + +// weekday_indexed + +class weekday_indexed +{ + unsigned char wd_ : 4; + unsigned char index_ : 4; + +public: + weekday_indexed() = default; + CONSTCD11 weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT; + + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 unsigned index() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; +CONSTCD11 bool operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_indexed& wdi); + +// weekday_last + +class weekday_last +{ + date::weekday wd_; + +public: + explicit CONSTCD11 weekday_last(const date::weekday& wd) NOEXCEPT; + + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT; +CONSTCD11 bool operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_last& wdl); + +namespace detail +{ + +struct unspecified_month_disambiguator {}; + +} // namespace detail + + // year_month + +class year_month +{ + date::year y_; + date::month m_; + +public: + year_month() = default; + CONSTCD11 year_month(const date::year& y, const date::month& m) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + + template + CONSTCD14 year_month& operator+=(const months& dm) NOEXCEPT; + template + CONSTCD14 year_month& operator-=(const months& dm) NOEXCEPT; + CONSTCD14 year_month& operator+=(const years& dy) NOEXCEPT; + CONSTCD14 year_month& operator-=(const years& dy) NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator!=(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator< (const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator> (const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator<=(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator>=(const year_month& x, const year_month& y) NOEXCEPT; + +template +CONSTCD14 year_month operator+(const year_month& ym, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month operator+(const months& dm, const year_month& ym) NOEXCEPT; +template +CONSTCD14 year_month operator-(const year_month& ym, const months& dm) NOEXCEPT; + +CONSTCD11 months operator-(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 year_month operator+(const year_month& ym, const years& dy) NOEXCEPT; +CONSTCD11 year_month operator+(const years& dy, const year_month& ym) NOEXCEPT; +CONSTCD11 year_month operator-(const year_month& ym, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month& ym); + +// month_day + +class month_day +{ + date::month m_; + date::day d_; + +public: + month_day() = default; + CONSTCD11 month_day(const date::month& m, const date::day& d) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::day day() const NOEXCEPT; + + CONSTCD14 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator< (const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator> (const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator<=(const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator>=(const month_day& x, const month_day& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day& md); + +// month_day_last + +class month_day_last +{ + date::month m_; + +public: + CONSTCD11 explicit month_day_last(const date::month& m) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator< (const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator> (const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day_last& mdl); + +// month_weekday + +class month_weekday +{ + date::month m_; + date::weekday_indexed wdi_; +public: + CONSTCD11 month_weekday(const date::month& m, + const date::weekday_indexed& wdi) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday& mwd); + +// month_weekday_last + +class month_weekday_last +{ + date::month m_; + date::weekday_last wdl_; + +public: + CONSTCD11 month_weekday_last(const date::month& m, + const date::weekday_last& wd) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 +bool operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; +CONSTCD11 +bool operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday_last& mwdl); + +// class year_month_day + +class year_month_day +{ + date::year y_; + date::month m_; + date::day d_; + +public: + year_month_day() = default; + CONSTCD11 year_month_day(const date::year& y, const date::month& m, + const date::day& d) NOEXCEPT; + CONSTCD14 year_month_day(const year_month_day_last& ymdl) NOEXCEPT; + + CONSTCD14 year_month_day(sys_days dp) NOEXCEPT; + CONSTCD14 explicit year_month_day(local_days dp) NOEXCEPT; + + template + CONSTCD14 year_month_day& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_day& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_day& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::day day() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD14 bool ok() const NOEXCEPT; + +private: + static CONSTCD14 year_month_day from_days(days dp) NOEXCEPT; + CONSTCD14 days to_days() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator< (const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator> (const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT; + +template +CONSTCD14 year_month_day operator+(const year_month_day& ymd, const months& dm) NOEXCEPT; +template +CONSTCD14 year_month_day operator+(const months& dm, const year_month_day& ymd) NOEXCEPT; +template +CONSTCD14 year_month_day operator-(const year_month_day& ymd, const months& dm) NOEXCEPT; +CONSTCD11 year_month_day operator+(const year_month_day& ymd, const years& dy) NOEXCEPT; +CONSTCD11 year_month_day operator+(const years& dy, const year_month_day& ymd) NOEXCEPT; +CONSTCD11 year_month_day operator-(const year_month_day& ymd, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day& ymd); + +// year_month_day_last + +class year_month_day_last +{ + date::year y_; + date::month_day_last mdl_; + +public: + CONSTCD11 year_month_day_last(const date::year& y, + const date::month_day_last& mdl) NOEXCEPT; + + template + CONSTCD14 year_month_day_last& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_day_last& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day_last& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_day_last& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::month_day_last month_day_last() const NOEXCEPT; + CONSTCD14 date::day day() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 +bool operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 +bool operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 +bool operator< (const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 +bool operator> (const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 +bool operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 +bool operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; + +template +CONSTCD14 +year_month_day_last +operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; + +template +CONSTCD14 +year_month_day_last +operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT; + +CONSTCD11 +year_month_day_last +operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; + +CONSTCD11 +year_month_day_last +operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT; + +template +CONSTCD14 +year_month_day_last +operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; + +CONSTCD11 +year_month_day_last +operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day_last& ymdl); + +// year_month_weekday + +class year_month_weekday +{ + date::year y_; + date::month m_; + date::weekday_indexed wdi_; + +public: + year_month_weekday() = default; + CONSTCD11 year_month_weekday(const date::year& y, const date::month& m, + const date::weekday_indexed& wdi) NOEXCEPT; + CONSTCD14 year_month_weekday(const sys_days& dp) NOEXCEPT; + CONSTCD14 explicit year_month_weekday(const local_days& dp) NOEXCEPT; + + template + CONSTCD14 year_month_weekday& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_weekday& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_weekday& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 unsigned index() const NOEXCEPT; + CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD14 bool ok() const NOEXCEPT; + +private: + static CONSTCD14 year_month_weekday from_days(days dp) NOEXCEPT; + CONSTCD14 days to_days() const NOEXCEPT; +}; + +CONSTCD11 +bool operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; +CONSTCD11 +bool operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; + +template +CONSTCD14 +year_month_weekday +operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; + +template +CONSTCD14 +year_month_weekday +operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT; + +template +CONSTCD14 +year_month_weekday +operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday& ymwdi); + +// year_month_weekday_last + +class year_month_weekday_last +{ + date::year y_; + date::month m_; + date::weekday_last wdl_; + +public: + CONSTCD11 year_month_weekday_last(const date::year& y, const date::month& m, + const date::weekday_last& wdl) NOEXCEPT; + + template + CONSTCD14 year_month_weekday_last& operator+=(const months& m) NOEXCEPT; + template + CONSTCD14 year_month_weekday_last& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday_last& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_weekday_last& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; + +private: + CONSTCD14 days to_days() const NOEXCEPT; +}; + +CONSTCD11 +bool +operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; + +CONSTCD11 +bool +operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; + +template +CONSTCD14 +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; + +template +CONSTCD14 +year_month_weekday_last +operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT; + +template +CONSTCD14 +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday_last& ymwdl); + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +inline namespace literals +{ + +CONSTCD11 date::day operator "" _d(unsigned long long d) NOEXCEPT; +CONSTCD11 date::year operator "" _y(unsigned long long y) NOEXCEPT; + +} // inline namespace literals +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) + + // CONSTDATA date::month January{1}; + // CONSTDATA date::month February{2}; + // CONSTDATA date::month March{3}; + // CONSTDATA date::month April{4}; + // CONSTDATA date::month May{5}; + // CONSTDATA date::month June{6}; + // CONSTDATA date::month July{7}; + // CONSTDATA date::month August{8}; + // CONSTDATA date::month September{9}; + // CONSTDATA date::month October{10}; + // CONSTDATA date::month November{11}; + // CONSTDATA date::month December{12}; + // + // CONSTDATA date::weekday Sunday{0u}; + // CONSTDATA date::weekday Monday{1u}; + // CONSTDATA date::weekday Tuesday{2u}; + // CONSTDATA date::weekday Wednesday{3u}; + // CONSTDATA date::weekday Thursday{4u}; + // CONSTDATA date::weekday Friday{5u}; + // CONSTDATA date::weekday Saturday{6u}; + +#if HAS_VOID_T + +template > +struct is_clock + : std::false_type +{}; + +template +struct is_clock> + : std::true_type +{}; + +#endif // HAS_VOID_T + +//----------------+ +// Implementation | +//----------------+ + +// utilities +namespace detail { + +template> +class save_istream +{ +protected: + std::basic_ios& is_; + CharT fill_; + std::ios::fmtflags flags_; + std::streamsize width_; + std::basic_ostream* tie_; + std::locale loc_; + +public: + ~save_istream() + { + is_.fill(fill_); + is_.flags(flags_); + is_.width(width_); + is_.imbue(loc_); + is_.tie(tie_); + } + + save_istream(const save_istream&) = delete; + save_istream& operator=(const save_istream&) = delete; + + explicit save_istream(std::basic_ios& is) + : is_(is) + , fill_(is.fill()) + , flags_(is.flags()) + , width_(is.width(0)) + , tie_(is.tie(nullptr)) + , loc_(is.getloc()) + { + if (tie_ != nullptr) + tie_->flush(); + } +}; + +template> +class save_ostream + : private save_istream +{ +public: + ~save_ostream() + { + if ((this->flags_ & std::ios::unitbuf) && +#if HAS_UNCAUGHT_EXCEPTIONS + std::uncaught_exceptions() == 0 && +#else + !std::uncaught_exception() && +#endif + this->is_.good()) + this->is_.rdbuf()->pubsync(); + } + + save_ostream(const save_ostream&) = delete; + save_ostream& operator=(const save_ostream&) = delete; + + explicit save_ostream(std::basic_ios& os) + : save_istream(os) + { + } +}; + +template +struct choose_trunc_type +{ + static const int digits = std::numeric_limits::digits; + using type = typename std::conditional + < + digits < 32, + std::int32_t, + typename std::conditional + < + digits < 64, + std::int64_t, +#ifdef __SIZEOF_INT128__ + __int128 +#else + std::int64_t +#endif + >::type + >::type; +}; + +template +CONSTCD11 +inline +typename std::enable_if +< + !std::chrono::treat_as_floating_point::value, + T +>::type +trunc(T t) NOEXCEPT +{ + return t; +} + +template +CONSTCD14 +inline +typename std::enable_if +< + std::chrono::treat_as_floating_point::value, + T +>::type +trunc(T t) NOEXCEPT +{ + using std::numeric_limits; + using I = typename choose_trunc_type::type; + CONSTDATA auto digits = numeric_limits::digits; + static_assert(digits < numeric_limits::digits, ""); + CONSTDATA auto max = I{1} << (digits-1); + CONSTDATA auto min = -max; + const auto negative = t < T{0}; + if (min <= t && t <= max && t != 0 && t == t) + { + t = static_cast(static_cast(t)); + if (t == 0 && negative) + t = -t; + } + return t; +} + +template +struct static_gcd +{ + static const std::intmax_t value = static_gcd::value; +}; + +template +struct static_gcd +{ + static const std::intmax_t value = Xp; +}; + +template <> +struct static_gcd<0, 0> +{ + static const std::intmax_t value = 1; +}; + +template +struct no_overflow +{ +private: + static const std::intmax_t gcd_n1_n2 = static_gcd::value; + static const std::intmax_t gcd_d1_d2 = static_gcd::value; + static const std::intmax_t n1 = R1::num / gcd_n1_n2; + static const std::intmax_t d1 = R1::den / gcd_d1_d2; + static const std::intmax_t n2 = R2::num / gcd_n1_n2; + static const std::intmax_t d2 = R2::den / gcd_d1_d2; +#ifdef __cpp_constexpr + static const std::intmax_t max = std::numeric_limits::max(); +#else + static const std::intmax_t max = LLONG_MAX; +#endif + + template + struct mul // overflow == false + { + static const std::intmax_t value = Xp * Yp; + }; + + template + struct mul + { + static const std::intmax_t value = 1; + }; + +public: + static const bool value = (n1 <= max / d2) && (n2 <= max / d1); + typedef std::ratio::value, + mul::value> type; +}; + +} // detail + + // trunc towards zero +template +CONSTCD11 +inline +typename std::enable_if +< + detail::no_overflow::value, + To +>::type +trunc(const std::chrono::duration& d) +{ + return To{detail::trunc(std::chrono::duration_cast(d).count())}; +} + +template +CONSTCD11 +inline +typename std::enable_if +< + !detail::no_overflow::value, + To +>::type +trunc(const std::chrono::duration& d) +{ + using std::chrono::duration_cast; + using std::chrono::duration; + using rep = typename std::common_type::type; + return To{detail::trunc(duration_cast(duration_cast>(d)).count())}; +} + +#ifndef HAS_CHRONO_ROUNDING +# if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190023918 || (_MSC_FULL_VER >= 190000000 && defined (__clang__))) +# define HAS_CHRONO_ROUNDING 1 +# elif defined(__cpp_lib_chrono) && __cplusplus > 201402 && __cpp_lib_chrono >= 201510 +# define HAS_CHRONO_ROUNDING 1 +# elif defined(_LIBCPP_VERSION) && __cplusplus > 201402 && _LIBCPP_VERSION >= 3800 +# define HAS_CHRONO_ROUNDING 1 +# else +# define HAS_CHRONO_ROUNDING 0 +# endif +#endif // HAS_CHRONO_ROUNDING + +#if HAS_CHRONO_ROUNDING == 0 + +// round down +template +CONSTCD14 +inline +typename std::enable_if +< + detail::no_overflow::value, + To +>::type +floor(const std::chrono::duration& d) +{ + auto t = trunc(d); + if (t > d) + return t - To{1}; + return t; +} + +template +CONSTCD14 +inline +typename std::enable_if +< + !detail::no_overflow::value, + To +>::type +floor(const std::chrono::duration& d) +{ + using rep = typename std::common_type::type; + return floor(floor>(d)); +} + +// round to nearest, to even on tie +template +CONSTCD14 +inline +To +round(const std::chrono::duration& d) +{ + auto t0 = floor(d); + auto t1 = t0 + To{1}; + if (t1 == To{0} && t0 < To{0}) + t1 = -t1; + auto diff0 = d - t0; + auto diff1 = t1 - d; + if (diff0 == diff1) + { + if (t0 - trunc(t0/2)*2 == To{0}) + return t0; + return t1; + } + if (diff0 < diff1) + return t0; + return t1; +} + +// round up +template +CONSTCD14 +inline +To +ceil(const std::chrono::duration& d) +{ + auto t = trunc(d); + if (t < d) + return t + To{1}; + return t; +} + +template ::is_signed +>::type> +CONSTCD11 +std::chrono::duration +abs(std::chrono::duration d) +{ + return d >= d.zero() ? d : -d; +} + +// round down +template +CONSTCD11 +inline +std::chrono::time_point +floor(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{date::floor(tp.time_since_epoch())}; +} + +// round to nearest, to even on tie +template +CONSTCD11 +inline +std::chrono::time_point +round(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{round(tp.time_since_epoch())}; +} + +// round up +template +CONSTCD11 +inline +std::chrono::time_point +ceil(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{ceil(tp.time_since_epoch())}; +} + +#else // HAS_CHRONO_ROUNDING == 1 + +using std::chrono::floor; +using std::chrono::ceil; +using std::chrono::round; +using std::chrono::abs; + +#endif // HAS_CHRONO_ROUNDING + +namespace detail +{ + +template +CONSTCD14 +inline +typename std::enable_if +< + !std::chrono::treat_as_floating_point::value, + To +>::type +round_i(const std::chrono::duration& d) +{ + return round(d); +} + +template +CONSTCD14 +inline +typename std::enable_if +< + std::chrono::treat_as_floating_point::value, + To +>::type +round_i(const std::chrono::duration& d) +{ + return d; +} + +template +CONSTCD11 +inline +std::chrono::time_point +round_i(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{round_i(tp.time_since_epoch())}; +} + +} // detail + + // trunc towards zero +template +CONSTCD11 +inline +std::chrono::time_point +trunc(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{trunc(tp.time_since_epoch())}; +} + +// day + +CONSTCD11 inline day::day(unsigned d) NOEXCEPT : d_(static_cast(d)) {} +CONSTCD14 inline day& day::operator++() NOEXCEPT {++d_; return *this;} +CONSTCD14 inline day day::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline day& day::operator--() NOEXCEPT {--d_; return *this;} +CONSTCD14 inline day day::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} +CONSTCD14 inline day& day::operator+=(const days& d) NOEXCEPT {*this = *this + d; return *this;} +CONSTCD14 inline day& day::operator-=(const days& d) NOEXCEPT {*this = *this - d; return *this;} +CONSTCD11 inline day::operator unsigned() const NOEXCEPT {return d_;} +CONSTCD11 inline bool day::ok() const NOEXCEPT {return 1 <= d_ && d_ <= 31;} + +CONSTCD11 +inline +bool +operator==(const day& x, const day& y) NOEXCEPT +{ + return static_cast(x) == static_cast(y); +} + +CONSTCD11 +inline +bool +operator!=(const day& x, const day& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const day& x, const day& y) NOEXCEPT +{ + return static_cast(x) < static_cast(y); +} + +CONSTCD11 +inline +bool +operator>(const day& x, const day& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const day& x, const day& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const day& x, const day& y) NOEXCEPT +{ + return !(x < y); +} + +CONSTCD11 +inline +days +operator-(const day& x, const day& y) NOEXCEPT +{ + return days{static_cast(static_cast(x) + - static_cast(y))}; +} + +CONSTCD11 +inline +day +operator+(const day& x, const days& y) NOEXCEPT +{ + return day{static_cast(x) + static_cast(y.count())}; +} + +CONSTCD11 +inline +day +operator+(const days& x, const day& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD11 +inline +day +operator-(const day& x, const days& y) NOEXCEPT +{ + return x + -y; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const day& d) +{ + detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(d); + if (!d.ok()) + os << " is not a valid day"; + return os; +} + +// month + +CONSTCD11 inline month::month(unsigned m) NOEXCEPT : m_(static_cast(m)) {} +CONSTCD14 inline month& month::operator++() NOEXCEPT {*this += months{1}; return *this;} +CONSTCD14 inline month month::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline month& month::operator--() NOEXCEPT {*this -= months{1}; return *this;} +CONSTCD14 inline month month::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} + +CONSTCD14 +inline +month& +month::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +CONSTCD14 +inline +month& +month::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD11 inline month::operator unsigned() const NOEXCEPT {return m_;} +CONSTCD11 inline bool month::ok() const NOEXCEPT {return 1 <= m_ && m_ <= 12;} + +CONSTCD11 +inline +bool +operator==(const month& x, const month& y) NOEXCEPT +{ + return static_cast(x) == static_cast(y); +} + +CONSTCD11 +inline +bool +operator!=(const month& x, const month& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const month& x, const month& y) NOEXCEPT +{ + return static_cast(x) < static_cast(y); +} + +CONSTCD11 +inline +bool +operator>(const month& x, const month& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const month& x, const month& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const month& x, const month& y) NOEXCEPT +{ + return !(x < y); +} + +CONSTCD14 +inline +months +operator-(const month& x, const month& y) NOEXCEPT +{ + auto const d = static_cast(x) - static_cast(y); + return months(d <= 11 ? d : d + 12); +} + +CONSTCD14 +inline +month +operator+(const month& x, const months& y) NOEXCEPT +{ + auto const mu = static_cast(static_cast(x)) + y.count() - 1; + auto const yr = (mu >= 0 ? mu : mu-11) / 12; + return month{static_cast(mu - yr * 12 + 1)}; +} + +CONSTCD14 +inline +month +operator+(const months& x, const month& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD14 +inline +month +operator-(const month& x, const months& y) NOEXCEPT +{ + return x + -y; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month& m) +{ + if (m.ok()) + { + CharT fmt[] = {'%', 'b', 0}; + os << format(os.getloc(), fmt, m); + } + else + os << static_cast(m) << " is not a valid month"; + return os; +} + +// year + +CONSTCD11 inline year::year(int y) NOEXCEPT : y_(static_cast(y)) {} +CONSTCD14 inline year& year::operator++() NOEXCEPT {++y_; return *this;} +CONSTCD14 inline year year::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline year& year::operator--() NOEXCEPT {--y_; return *this;} +CONSTCD14 inline year year::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} +CONSTCD14 inline year& year::operator+=(const years& y) NOEXCEPT {*this = *this + y; return *this;} +CONSTCD14 inline year& year::operator-=(const years& y) NOEXCEPT {*this = *this - y; return *this;} +CONSTCD11 inline year year::operator-() const NOEXCEPT {return year{-y_};} +CONSTCD11 inline year year::operator+() const NOEXCEPT {return *this;} + +CONSTCD11 +inline +bool +year::is_leap() const NOEXCEPT +{ + return y_ % 4 == 0 && (y_ % 100 != 0 || y_ % 400 == 0); +} + +CONSTCD11 inline year::operator int() const NOEXCEPT {return y_;} + +CONSTCD11 +inline +bool +year::ok() const NOEXCEPT +{ + return y_ != std::numeric_limits::min(); +} + +CONSTCD11 +inline +bool +operator==(const year& x, const year& y) NOEXCEPT +{ + return static_cast(x) == static_cast(y); +} + +CONSTCD11 +inline +bool +operator!=(const year& x, const year& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year& x, const year& y) NOEXCEPT +{ + return static_cast(x) < static_cast(y); +} + +CONSTCD11 +inline +bool +operator>(const year& x, const year& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year& x, const year& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year& x, const year& y) NOEXCEPT +{ + return !(x < y); +} + +CONSTCD11 +inline +years +operator-(const year& x, const year& y) NOEXCEPT +{ + return years{static_cast(x) - static_cast(y)}; +} + +CONSTCD11 +inline +year +operator+(const year& x, const years& y) NOEXCEPT +{ + return year{static_cast(x) + y.count()}; +} + +CONSTCD11 +inline +year +operator+(const years& x, const year& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD11 +inline +year +operator-(const year& x, const years& y) NOEXCEPT +{ + return year{static_cast(x) - y.count()}; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year& y) +{ + detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::internal); + os.width(4 + (y < year{0})); + os.imbue(std::locale::classic()); + os << static_cast(y); + if (!y.ok()) + os << " is not a valid year"; + return os; +} + +// weekday + +CONSTCD14 +inline +unsigned char +weekday::weekday_from_days(int z) NOEXCEPT +{ + auto u = static_cast(z); + return static_cast(z >= -4 ? (u+4) % 7 : u % 7); +} + +CONSTCD11 +inline +weekday::weekday(unsigned wd) NOEXCEPT + : wd_(static_cast(wd != 7 ? wd : 0)) +{} + +CONSTCD14 +inline +weekday::weekday(const sys_days& dp) NOEXCEPT + : wd_(weekday_from_days(dp.time_since_epoch().count())) +{} + +CONSTCD14 +inline +weekday::weekday(const local_days& dp) NOEXCEPT + : wd_(weekday_from_days(dp.time_since_epoch().count())) +{} + +CONSTCD14 inline weekday& weekday::operator++() NOEXCEPT {*this += days{1}; return *this;} +CONSTCD14 inline weekday weekday::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline weekday& weekday::operator--() NOEXCEPT {*this -= days{1}; return *this;} +CONSTCD14 inline weekday weekday::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} + +CONSTCD14 +inline +weekday& +weekday::operator+=(const days& d) NOEXCEPT +{ + *this = *this + d; + return *this; +} + +CONSTCD14 +inline +weekday& +weekday::operator-=(const days& d) NOEXCEPT +{ + *this = *this - d; + return *this; +} + +CONSTCD11 inline bool weekday::ok() const NOEXCEPT {return wd_ <= 6;} + +CONSTCD11 +inline +unsigned weekday::c_encoding() const NOEXCEPT +{ + return unsigned{wd_}; +} + +CONSTCD11 +inline +unsigned weekday::iso_encoding() const NOEXCEPT +{ + return unsigned{((wd_ == 0u) ? 7u : wd_)}; +} + +CONSTCD11 +inline +bool +operator==(const weekday& x, const weekday& y) NOEXCEPT +{ + return x.wd_ == y.wd_; +} + +CONSTCD11 +inline +bool +operator!=(const weekday& x, const weekday& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD14 +inline +days +operator-(const weekday& x, const weekday& y) NOEXCEPT +{ + auto const wdu = x.wd_ - y.wd_; + auto const wk = (wdu >= 0 ? wdu : wdu-6) / 7; + return days{wdu - wk * 7}; +} + +CONSTCD14 +inline +weekday +operator+(const weekday& x, const days& y) NOEXCEPT +{ + auto const wdu = static_cast(static_cast(x.wd_)) + y.count(); + auto const wk = (wdu >= 0 ? wdu : wdu-6) / 7; + return weekday{static_cast(wdu - wk * 7)}; +} + +CONSTCD14 +inline +weekday +operator+(const days& x, const weekday& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD14 +inline +weekday +operator-(const weekday& x, const days& y) NOEXCEPT +{ + return x + -y; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday& wd) +{ + if (wd.ok()) + { + CharT fmt[] = {'%', 'a', 0}; + os << format(fmt, wd); + } + else + os << static_cast(wd.wd_) << " is not a valid weekday"; + return os; +} + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +inline namespace literals +{ + +CONSTCD11 +inline +date::day +operator "" _d(unsigned long long d) NOEXCEPT +{ + return date::day{static_cast(d)}; +} + +CONSTCD11 +inline +date::year +operator "" _y(unsigned long long y) NOEXCEPT +{ + return date::year(static_cast(y)); +} +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) + +CONSTDATA date::last_spec last{}; + +CONSTDATA date::month jan{1}; +CONSTDATA date::month feb{2}; +CONSTDATA date::month mar{3}; +CONSTDATA date::month apr{4}; +CONSTDATA date::month may{5}; +CONSTDATA date::month jun{6}; +CONSTDATA date::month jul{7}; +CONSTDATA date::month aug{8}; +CONSTDATA date::month sep{9}; +CONSTDATA date::month oct{10}; +CONSTDATA date::month nov{11}; +CONSTDATA date::month dec{12}; + +CONSTDATA date::weekday sun{0u}; +CONSTDATA date::weekday mon{1u}; +CONSTDATA date::weekday tue{2u}; +CONSTDATA date::weekday wed{3u}; +CONSTDATA date::weekday thu{4u}; +CONSTDATA date::weekday fri{5u}; +CONSTDATA date::weekday sat{6u}; + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +} // inline namespace literals +#endif + +CONSTDATA date::month January{1}; +CONSTDATA date::month February{2}; +CONSTDATA date::month March{3}; +CONSTDATA date::month April{4}; +CONSTDATA date::month May{5}; +CONSTDATA date::month June{6}; +CONSTDATA date::month July{7}; +CONSTDATA date::month August{8}; +CONSTDATA date::month September{9}; +CONSTDATA date::month October{10}; +CONSTDATA date::month November{11}; +CONSTDATA date::month December{12}; + +CONSTDATA date::weekday Monday{1}; +CONSTDATA date::weekday Tuesday{2}; +CONSTDATA date::weekday Wednesday{3}; +CONSTDATA date::weekday Thursday{4}; +CONSTDATA date::weekday Friday{5}; +CONSTDATA date::weekday Saturday{6}; +CONSTDATA date::weekday Sunday{7}; + +// weekday_indexed + +CONSTCD11 +inline +weekday +weekday_indexed::weekday() const NOEXCEPT +{ + return date::weekday{static_cast(wd_)}; +} + +CONSTCD11 inline unsigned weekday_indexed::index() const NOEXCEPT {return index_;} + +CONSTCD11 +inline +bool +weekday_indexed::ok() const NOEXCEPT +{ + return weekday().ok() && 1 <= index_ && index_ <= 5; +} + +#ifdef __GNUC__ +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wconversion" +#endif // __GNUC__ + +CONSTCD11 +inline +weekday_indexed::weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT + : wd_(static_cast(static_cast(wd.wd_))) + , index_(static_cast(index)) +{} + +#ifdef __GNUC__ +# pragma GCC diagnostic pop +#endif // __GNUC__ + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_indexed& wdi) +{ + os << wdi.weekday() << '[' << wdi.index(); + if (!(1 <= wdi.index() && wdi.index() <= 5)) + os << " is not a valid index"; + os << ']'; + return os; +} + +CONSTCD11 +inline +weekday_indexed +weekday::operator[](unsigned index) const NOEXCEPT +{ + return {*this, index}; +} + +CONSTCD11 +inline +bool +operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT +{ + return x.weekday() == y.weekday() && x.index() == y.index(); +} + +CONSTCD11 +inline +bool +operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT +{ + return !(x == y); +} + +// weekday_last + +CONSTCD11 inline date::weekday weekday_last::weekday() const NOEXCEPT {return wd_;} +CONSTCD11 inline bool weekday_last::ok() const NOEXCEPT {return wd_.ok();} +CONSTCD11 inline weekday_last::weekday_last(const date::weekday& wd) NOEXCEPT : wd_(wd) {} + +CONSTCD11 +inline +bool +operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT +{ + return x.weekday() == y.weekday(); +} + +CONSTCD11 +inline +bool +operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_last& wdl) +{ + return os << wdl.weekday() << "[last]"; +} + +CONSTCD11 +inline +weekday_last +weekday::operator[](last_spec) const NOEXCEPT +{ + return weekday_last{*this}; +} + +// year_month + +CONSTCD11 +inline +year_month::year_month(const date::year& y, const date::month& m) NOEXCEPT + : y_(y) + , m_(m) +{} + +CONSTCD11 inline year year_month::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month::month() const NOEXCEPT {return m_;} +CONSTCD11 inline bool year_month::ok() const NOEXCEPT {return y_.ok() && m_.ok();} + +template +CONSTCD14 +inline +year_month& +year_month::operator+=(const months& dm) NOEXCEPT +{ + *this = *this + dm; + return *this; +} + +template +CONSTCD14 +inline +year_month& +year_month::operator-=(const months& dm) NOEXCEPT +{ + *this = *this - dm; + return *this; +} + +CONSTCD14 +inline +year_month& +year_month::operator+=(const years& dy) NOEXCEPT +{ + *this = *this + dy; + return *this; +} + +CONSTCD14 +inline +year_month& +year_month::operator-=(const years& dy) NOEXCEPT +{ + *this = *this - dy; + return *this; +} + +CONSTCD11 +inline +bool +operator==(const year_month& x, const year_month& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month& x, const year_month& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year_month& x, const year_month& y) NOEXCEPT +{ + return x.year() < y.year() ? true + : (x.year() > y.year() ? false + : (x.month() < y.month())); +} + +CONSTCD11 +inline +bool +operator>(const year_month& x, const year_month& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year_month& x, const year_month& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year_month& x, const year_month& y) NOEXCEPT +{ + return !(x < y); +} + +template +CONSTCD14 +inline +year_month +operator+(const year_month& ym, const months& dm) NOEXCEPT +{ + auto dmi = static_cast(static_cast(ym.month())) - 1 + dm.count(); + auto dy = (dmi >= 0 ? dmi : dmi-11) / 12; + dmi = dmi - dy * 12 + 1; + return (ym.year() + years(dy)) / month(static_cast(dmi)); +} + +template +CONSTCD14 +inline +year_month +operator+(const months& dm, const year_month& ym) NOEXCEPT +{ + return ym + dm; +} + +template +CONSTCD14 +inline +year_month +operator-(const year_month& ym, const months& dm) NOEXCEPT +{ + return ym + -dm; +} + +CONSTCD11 +inline +months +operator-(const year_month& x, const year_month& y) NOEXCEPT +{ + return (x.year() - y.year()) + + months(static_cast(x.month()) - static_cast(y.month())); +} + +CONSTCD11 +inline +year_month +operator+(const year_month& ym, const years& dy) NOEXCEPT +{ + return (ym.year() + dy) / ym.month(); +} + +CONSTCD11 +inline +year_month +operator+(const years& dy, const year_month& ym) NOEXCEPT +{ + return ym + dy; +} + +CONSTCD11 +inline +year_month +operator-(const year_month& ym, const years& dy) NOEXCEPT +{ + return ym + -dy; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month& ym) +{ + return os << ym.year() << '/' << ym.month(); +} + +// month_day + +CONSTCD11 +inline +month_day::month_day(const date::month& m, const date::day& d) NOEXCEPT + : m_(m) + , d_(d) +{} + +CONSTCD11 inline date::month month_day::month() const NOEXCEPT {return m_;} +CONSTCD11 inline date::day month_day::day() const NOEXCEPT {return d_;} + +CONSTCD14 +inline +bool +month_day::ok() const NOEXCEPT +{ + CONSTDATA date::day d[] = + { + date::day(31), date::day(29), date::day(31), + date::day(30), date::day(31), date::day(30), + date::day(31), date::day(31), date::day(30), + date::day(31), date::day(30), date::day(31) + }; + return m_.ok() && date::day{1} <= d_ && d_ <= d[static_cast(m_)-1]; +} + +CONSTCD11 +inline +bool +operator==(const month_day& x, const month_day& y) NOEXCEPT +{ + return x.month() == y.month() && x.day() == y.day(); +} + +CONSTCD11 +inline +bool +operator!=(const month_day& x, const month_day& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const month_day& x, const month_day& y) NOEXCEPT +{ + return x.month() < y.month() ? true + : (x.month() > y.month() ? false + : (x.day() < y.day())); +} + +CONSTCD11 +inline +bool +operator>(const month_day& x, const month_day& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const month_day& x, const month_day& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const month_day& x, const month_day& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day& md) +{ + return os << md.month() << '/' << md.day(); +} + +// month_day_last + +CONSTCD11 inline month month_day_last::month() const NOEXCEPT {return m_;} +CONSTCD11 inline bool month_day_last::ok() const NOEXCEPT {return m_.ok();} +CONSTCD11 inline month_day_last::month_day_last(const date::month& m) NOEXCEPT : m_(m) {} + +CONSTCD11 +inline +bool +operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return x.month() == y.month(); +} + +CONSTCD11 +inline +bool +operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return x.month() < y.month(); +} + +CONSTCD11 +inline +bool +operator>(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day_last& mdl) +{ + return os << mdl.month() << "/last"; +} + +// month_weekday + +CONSTCD11 +inline +month_weekday::month_weekday(const date::month& m, + const date::weekday_indexed& wdi) NOEXCEPT + : m_(m) + , wdi_(wdi) +{} + +CONSTCD11 inline month month_weekday::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday_indexed +month_weekday::weekday_indexed() const NOEXCEPT +{ + return wdi_; +} + +CONSTCD11 +inline +bool +month_weekday::ok() const NOEXCEPT +{ + return m_.ok() && wdi_.ok(); +} + +CONSTCD11 +inline +bool +operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT +{ + return x.month() == y.month() && x.weekday_indexed() == y.weekday_indexed(); +} + +CONSTCD11 +inline +bool +operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday& mwd) +{ + return os << mwd.month() << '/' << mwd.weekday_indexed(); +} + +// month_weekday_last + +CONSTCD11 +inline +month_weekday_last::month_weekday_last(const date::month& m, + const date::weekday_last& wdl) NOEXCEPT + : m_(m) + , wdl_(wdl) +{} + +CONSTCD11 inline month month_weekday_last::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday_last +month_weekday_last::weekday_last() const NOEXCEPT +{ + return wdl_; +} + +CONSTCD11 +inline +bool +month_weekday_last::ok() const NOEXCEPT +{ + return m_.ok() && wdl_.ok(); +} + +CONSTCD11 +inline +bool +operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT +{ + return x.month() == y.month() && x.weekday_last() == y.weekday_last(); +} + +CONSTCD11 +inline +bool +operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday_last& mwdl) +{ + return os << mwdl.month() << '/' << mwdl.weekday_last(); +} + +// year_month_day_last + +CONSTCD11 +inline +year_month_day_last::year_month_day_last(const date::year& y, + const date::month_day_last& mdl) NOEXCEPT + : y_(y) + , mdl_(mdl) +{} + +template +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +template +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD11 inline year year_month_day_last::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_day_last::month() const NOEXCEPT {return mdl_.month();} + +CONSTCD11 +inline +month_day_last +year_month_day_last::month_day_last() const NOEXCEPT +{ + return mdl_; +} + +CONSTCD14 +inline +day +year_month_day_last::day() const NOEXCEPT +{ + CONSTDATA date::day d[] = + { + date::day(31), date::day(28), date::day(31), + date::day(30), date::day(31), date::day(30), + date::day(31), date::day(31), date::day(30), + date::day(31), date::day(30), date::day(31) + }; + return (month() != February || !y_.is_leap()) && mdl_.ok() ? + d[static_cast(month()) - 1] : date::day{29}; +} + +CONSTCD14 +inline +year_month_day_last::operator sys_days() const NOEXCEPT +{ + return sys_days(year()/month()/day()); +} + +CONSTCD14 +inline +year_month_day_last::operator local_days() const NOEXCEPT +{ + return local_days(year()/month()/day()); +} + +CONSTCD11 +inline +bool +year_month_day_last::ok() const NOEXCEPT +{ + return y_.ok() && mdl_.ok(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return x.year() == y.year() && x.month_day_last() == y.month_day_last(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return x.year() < y.year() ? true + : (x.year() > y.year() ? false + : (x.month_day_last() < y.month_day_last())); +} + +CONSTCD11 +inline +bool +operator>(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day_last& ymdl) +{ + return os << ymdl.year() << '/' << ymdl.month_day_last(); +} + +template +CONSTCD14 +inline +year_month_day_last +operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT +{ + return (ymdl.year() / ymdl.month() + dm) / last; +} + +template +CONSTCD14 +inline +year_month_day_last +operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT +{ + return ymdl + dm; +} + +template +CONSTCD14 +inline +year_month_day_last +operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT +{ + return ymdl + (-dm); +} + +CONSTCD11 +inline +year_month_day_last +operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT +{ + return {ymdl.year()+dy, ymdl.month_day_last()}; +} + +CONSTCD11 +inline +year_month_day_last +operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT +{ + return ymdl + dy; +} + +CONSTCD11 +inline +year_month_day_last +operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT +{ + return ymdl + (-dy); +} + +// year_month_day + +CONSTCD11 +inline +year_month_day::year_month_day(const date::year& y, const date::month& m, + const date::day& d) NOEXCEPT + : y_(y) + , m_(m) + , d_(d) +{} + +CONSTCD14 +inline +year_month_day::year_month_day(const year_month_day_last& ymdl) NOEXCEPT + : y_(ymdl.year()) + , m_(ymdl.month()) + , d_(ymdl.day()) +{} + +CONSTCD14 +inline +year_month_day::year_month_day(sys_days dp) NOEXCEPT + : year_month_day(from_days(dp.time_since_epoch())) +{} + +CONSTCD14 +inline +year_month_day::year_month_day(local_days dp) NOEXCEPT + : year_month_day(from_days(dp.time_since_epoch())) +{} + +CONSTCD11 inline year year_month_day::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_day::month() const NOEXCEPT {return m_;} +CONSTCD11 inline day year_month_day::day() const NOEXCEPT {return d_;} + +template +CONSTCD14 +inline +year_month_day& +year_month_day::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +template +CONSTCD14 +inline +year_month_day& +year_month_day::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_day& +year_month_day::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_day& +year_month_day::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD14 +inline +days +year_month_day::to_days() const NOEXCEPT +{ + static_assert(std::numeric_limits::digits >= 18, + "This algorithm has not been ported to a 16 bit unsigned integer"); + static_assert(std::numeric_limits::digits >= 20, + "This algorithm has not been ported to a 16 bit signed integer"); + auto const y = static_cast(y_) - (m_ <= February); + auto const m = static_cast(m_); + auto const d = static_cast(d_); + auto const era = (y >= 0 ? y : y-399) / 400; + auto const yoe = static_cast(y - era * 400); // [0, 399] + auto const doy = (153*(m > 2 ? m-3 : m+9) + 2)/5 + d-1; // [0, 365] + auto const doe = yoe * 365 + yoe/4 - yoe/100 + doy; // [0, 146096] + return days{era * 146097 + static_cast(doe) - 719468}; +} + +CONSTCD14 +inline +year_month_day::operator sys_days() const NOEXCEPT +{ + return sys_days{to_days()}; +} + +CONSTCD14 +inline +year_month_day::operator local_days() const NOEXCEPT +{ + return local_days{to_days()}; +} + +CONSTCD14 +inline +bool +year_month_day::ok() const NOEXCEPT +{ + if (!(y_.ok() && m_.ok())) + return false; + return date::day{1} <= d_ && d_ <= (y_ / m_ / last).day(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month() && x.day() == y.day(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return x.year() < y.year() ? true + : (x.year() > y.year() ? false + : (x.month() < y.month() ? true + : (x.month() > y.month() ? false + : (x.day() < y.day())))); +} + +CONSTCD11 +inline +bool +operator>(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day& ymd) +{ + detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.imbue(std::locale::classic()); + os << ymd.year() << '-'; + os.width(2); + os << static_cast(ymd.month()) << '-'; + os << ymd.day(); + if (!ymd.ok()) + os << " is not a valid date"; + return os; +} + +CONSTCD14 +inline +year_month_day +year_month_day::from_days(days dp) NOEXCEPT +{ + static_assert(std::numeric_limits::digits >= 18, + "This algorithm has not been ported to a 16 bit unsigned integer"); + static_assert(std::numeric_limits::digits >= 20, + "This algorithm has not been ported to a 16 bit signed integer"); + auto const z = dp.count() + 719468; + auto const era = (z >= 0 ? z : z - 146096) / 146097; + auto const doe = static_cast(z - era * 146097); // [0, 146096] + auto const yoe = (doe - doe/1460 + doe/36524 - doe/146096) / 365; // [0, 399] + auto const y = static_cast(yoe) + era * 400; + auto const doy = doe - (365*yoe + yoe/4 - yoe/100); // [0, 365] + auto const mp = (5*doy + 2)/153; // [0, 11] + auto const d = doy - (153*mp+2)/5 + 1; // [1, 31] + auto const m = mp < 10 ? mp+3 : mp-9; // [1, 12] + return year_month_day{date::year{y + (m <= 2)}, date::month(m), date::day(d)}; +} + +template +CONSTCD14 +inline +year_month_day +operator+(const year_month_day& ymd, const months& dm) NOEXCEPT +{ + return (ymd.year() / ymd.month() + dm) / ymd.day(); +} + +template +CONSTCD14 +inline +year_month_day +operator+(const months& dm, const year_month_day& ymd) NOEXCEPT +{ + return ymd + dm; +} + +template +CONSTCD14 +inline +year_month_day +operator-(const year_month_day& ymd, const months& dm) NOEXCEPT +{ + return ymd + (-dm); +} + +CONSTCD11 +inline +year_month_day +operator+(const year_month_day& ymd, const years& dy) NOEXCEPT +{ + return (ymd.year() + dy) / ymd.month() / ymd.day(); +} + +CONSTCD11 +inline +year_month_day +operator+(const years& dy, const year_month_day& ymd) NOEXCEPT +{ + return ymd + dy; +} + +CONSTCD11 +inline +year_month_day +operator-(const year_month_day& ymd, const years& dy) NOEXCEPT +{ + return ymd + (-dy); +} + +// year_month_weekday + +CONSTCD11 +inline +year_month_weekday::year_month_weekday(const date::year& y, const date::month& m, + const date::weekday_indexed& wdi) + NOEXCEPT + : y_(y) + , m_(m) + , wdi_(wdi) +{} + +CONSTCD14 +inline +year_month_weekday::year_month_weekday(const sys_days& dp) NOEXCEPT + : year_month_weekday(from_days(dp.time_since_epoch())) +{} + +CONSTCD14 +inline +year_month_weekday::year_month_weekday(const local_days& dp) NOEXCEPT + : year_month_weekday(from_days(dp.time_since_epoch())) +{} + +template +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +template +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD11 inline year year_month_weekday::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_weekday::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday +year_month_weekday::weekday() const NOEXCEPT +{ + return wdi_.weekday(); +} + +CONSTCD11 +inline +unsigned +year_month_weekday::index() const NOEXCEPT +{ + return wdi_.index(); +} + +CONSTCD11 +inline +weekday_indexed +year_month_weekday::weekday_indexed() const NOEXCEPT +{ + return wdi_; +} + +CONSTCD14 +inline +year_month_weekday::operator sys_days() const NOEXCEPT +{ + return sys_days{to_days()}; +} + +CONSTCD14 +inline +year_month_weekday::operator local_days() const NOEXCEPT +{ + return local_days{to_days()}; +} + +CONSTCD14 +inline +bool +year_month_weekday::ok() const NOEXCEPT +{ + if (!y_.ok() || !m_.ok() || !wdi_.weekday().ok() || wdi_.index() < 1) + return false; + if (wdi_.index() <= 4) + return true; + auto d2 = wdi_.weekday() - date::weekday(static_cast(y_/m_/1)) + + days((wdi_.index()-1)*7 + 1); + return static_cast(d2.count()) <= static_cast((y_/m_/last).day()); +} + +CONSTCD14 +inline +year_month_weekday +year_month_weekday::from_days(days d) NOEXCEPT +{ + sys_days dp{d}; + auto const wd = date::weekday(dp); + auto const ymd = year_month_day(dp); + return {ymd.year(), ymd.month(), wd[(static_cast(ymd.day())-1)/7+1]}; +} + +CONSTCD14 +inline +days +year_month_weekday::to_days() const NOEXCEPT +{ + auto d = sys_days(y_/m_/1); + return (d + (wdi_.weekday() - date::weekday(d) + days{(wdi_.index()-1)*7}) + ).time_since_epoch(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month() && + x.weekday_indexed() == y.weekday_indexed(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday& ymwdi) +{ + return os << ymwdi.year() << '/' << ymwdi.month() + << '/' << ymwdi.weekday_indexed(); +} + +template +CONSTCD14 +inline +year_month_weekday +operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT +{ + return (ymwd.year() / ymwd.month() + dm) / ymwd.weekday_indexed(); +} + +template +CONSTCD14 +inline +year_month_weekday +operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT +{ + return ymwd + dm; +} + +template +CONSTCD14 +inline +year_month_weekday +operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT +{ + return ymwd + (-dm); +} + +CONSTCD11 +inline +year_month_weekday +operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT +{ + return {ymwd.year()+dy, ymwd.month(), ymwd.weekday_indexed()}; +} + +CONSTCD11 +inline +year_month_weekday +operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT +{ + return ymwd + dy; +} + +CONSTCD11 +inline +year_month_weekday +operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT +{ + return ymwd + (-dy); +} + +// year_month_weekday_last + +CONSTCD11 +inline +year_month_weekday_last::year_month_weekday_last(const date::year& y, + const date::month& m, + const date::weekday_last& wdl) NOEXCEPT + : y_(y) + , m_(m) + , wdl_(wdl) +{} + +template +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +template +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD11 inline year year_month_weekday_last::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_weekday_last::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday +year_month_weekday_last::weekday() const NOEXCEPT +{ + return wdl_.weekday(); +} + +CONSTCD11 +inline +weekday_last +year_month_weekday_last::weekday_last() const NOEXCEPT +{ + return wdl_; +} + +CONSTCD14 +inline +year_month_weekday_last::operator sys_days() const NOEXCEPT +{ + return sys_days{to_days()}; +} + +CONSTCD14 +inline +year_month_weekday_last::operator local_days() const NOEXCEPT +{ + return local_days{to_days()}; +} + +CONSTCD11 +inline +bool +year_month_weekday_last::ok() const NOEXCEPT +{ + return y_.ok() && m_.ok() && wdl_.ok(); +} + +CONSTCD14 +inline +days +year_month_weekday_last::to_days() const NOEXCEPT +{ + auto const d = sys_days(y_/m_/last); + return (d - (date::weekday{d} - wdl_.weekday())).time_since_epoch(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month() && + x.weekday_last() == y.weekday_last(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday_last& ymwdl) +{ + return os << ymwdl.year() << '/' << ymwdl.month() << '/' << ymwdl.weekday_last(); +} + +template +CONSTCD14 +inline +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT +{ + return (ymwdl.year() / ymwdl.month() + dm) / ymwdl.weekday_last(); +} + +template +CONSTCD14 +inline +year_month_weekday_last +operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT +{ + return ymwdl + dm; +} + +template +CONSTCD14 +inline +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT +{ + return ymwdl + (-dm); +} + +CONSTCD11 +inline +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT +{ + return {ymwdl.year()+dy, ymwdl.month(), ymwdl.weekday_last()}; +} + +CONSTCD11 +inline +year_month_weekday_last +operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT +{ + return ymwdl + dy; +} + +CONSTCD11 +inline +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT +{ + return ymwdl + (-dy); +} + +// year_month from operator/() + +CONSTCD11 +inline +year_month +operator/(const year& y, const month& m) NOEXCEPT +{ + return {y, m}; +} + +CONSTCD11 +inline +year_month +operator/(const year& y, int m) NOEXCEPT +{ + return y / month(static_cast(m)); +} + +// month_day from operator/() + +CONSTCD11 +inline +month_day +operator/(const month& m, const day& d) NOEXCEPT +{ + return {m, d}; +} + +CONSTCD11 +inline +month_day +operator/(const day& d, const month& m) NOEXCEPT +{ + return m / d; +} + +CONSTCD11 +inline +month_day +operator/(const month& m, int d) NOEXCEPT +{ + return m / day(static_cast(d)); +} + +CONSTCD11 +inline +month_day +operator/(int m, const day& d) NOEXCEPT +{ + return month(static_cast(m)) / d; +} + +CONSTCD11 inline month_day operator/(const day& d, int m) NOEXCEPT {return m / d;} + +// month_day_last from operator/() + +CONSTCD11 +inline +month_day_last +operator/(const month& m, last_spec) NOEXCEPT +{ + return month_day_last{m}; +} + +CONSTCD11 +inline +month_day_last +operator/(last_spec, const month& m) NOEXCEPT +{ + return m/last; +} + +CONSTCD11 +inline +month_day_last +operator/(int m, last_spec) NOEXCEPT +{ + return month(static_cast(m))/last; +} + +CONSTCD11 +inline +month_day_last +operator/(last_spec, int m) NOEXCEPT +{ + return m/last; +} + +// month_weekday from operator/() + +CONSTCD11 +inline +month_weekday +operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT +{ + return {m, wdi}; +} + +CONSTCD11 +inline +month_weekday +operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT +{ + return m / wdi; +} + +CONSTCD11 +inline +month_weekday +operator/(int m, const weekday_indexed& wdi) NOEXCEPT +{ + return month(static_cast(m)) / wdi; +} + +CONSTCD11 +inline +month_weekday +operator/(const weekday_indexed& wdi, int m) NOEXCEPT +{ + return m / wdi; +} + +// month_weekday_last from operator/() + +CONSTCD11 +inline +month_weekday_last +operator/(const month& m, const weekday_last& wdl) NOEXCEPT +{ + return {m, wdl}; +} + +CONSTCD11 +inline +month_weekday_last +operator/(const weekday_last& wdl, const month& m) NOEXCEPT +{ + return m / wdl; +} + +CONSTCD11 +inline +month_weekday_last +operator/(int m, const weekday_last& wdl) NOEXCEPT +{ + return month(static_cast(m)) / wdl; +} + +CONSTCD11 +inline +month_weekday_last +operator/(const weekday_last& wdl, int m) NOEXCEPT +{ + return m / wdl; +} + +// year_month_day from operator/() + +CONSTCD11 +inline +year_month_day +operator/(const year_month& ym, const day& d) NOEXCEPT +{ + return {ym.year(), ym.month(), d}; +} + +CONSTCD11 +inline +year_month_day +operator/(const year_month& ym, int d) NOEXCEPT +{ + return ym / day(static_cast(d)); +} + +CONSTCD11 +inline +year_month_day +operator/(const year& y, const month_day& md) NOEXCEPT +{ + return y / md.month() / md.day(); +} + +CONSTCD11 +inline +year_month_day +operator/(int y, const month_day& md) NOEXCEPT +{ + return year(y) / md; +} + +CONSTCD11 +inline +year_month_day +operator/(const month_day& md, const year& y) NOEXCEPT +{ + return y / md; +} + +CONSTCD11 +inline +year_month_day +operator/(const month_day& md, int y) NOEXCEPT +{ + return year(y) / md; +} + +// year_month_day_last from operator/() + +CONSTCD11 +inline +year_month_day_last +operator/(const year_month& ym, last_spec) NOEXCEPT +{ + return {ym.year(), month_day_last{ym.month()}}; +} + +CONSTCD11 +inline +year_month_day_last +operator/(const year& y, const month_day_last& mdl) NOEXCEPT +{ + return {y, mdl}; +} + +CONSTCD11 +inline +year_month_day_last +operator/(int y, const month_day_last& mdl) NOEXCEPT +{ + return year(y) / mdl; +} + +CONSTCD11 +inline +year_month_day_last +operator/(const month_day_last& mdl, const year& y) NOEXCEPT +{ + return y / mdl; +} + +CONSTCD11 +inline +year_month_day_last +operator/(const month_day_last& mdl, int y) NOEXCEPT +{ + return year(y) / mdl; +} + +// year_month_weekday from operator/() + +CONSTCD11 +inline +year_month_weekday +operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT +{ + return {ym.year(), ym.month(), wdi}; +} + +CONSTCD11 +inline +year_month_weekday +operator/(const year& y, const month_weekday& mwd) NOEXCEPT +{ + return {y, mwd.month(), mwd.weekday_indexed()}; +} + +CONSTCD11 +inline +year_month_weekday +operator/(int y, const month_weekday& mwd) NOEXCEPT +{ + return year(y) / mwd; +} + +CONSTCD11 +inline +year_month_weekday +operator/(const month_weekday& mwd, const year& y) NOEXCEPT +{ + return y / mwd; +} + +CONSTCD11 +inline +year_month_weekday +operator/(const month_weekday& mwd, int y) NOEXCEPT +{ + return year(y) / mwd; +} + +// year_month_weekday_last from operator/() + +CONSTCD11 +inline +year_month_weekday_last +operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT +{ + return {ym.year(), ym.month(), wdl}; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT +{ + return {y, mwdl.month(), mwdl.weekday_last()}; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(int y, const month_weekday_last& mwdl) NOEXCEPT +{ + return year(y) / mwdl; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT +{ + return y / mwdl; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(const month_weekday_last& mwdl, int y) NOEXCEPT +{ + return year(y) / mwdl; +} + +template +struct fields; + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev = nullptr, + const std::chrono::seconds* offset_sec = nullptr); + +template +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr); + +// hh_mm_ss + +namespace detail +{ + +struct undocumented {explicit undocumented() = default;}; + +// width::value is the number of fractional decimal digits in 1/n +// width<0>::value and width<1>::value are defined to be 0 +// If 1/n takes more than 18 fractional decimal digits, +// the result is truncated to 19. +// Example: width<2>::value == 1 +// Example: width<3>::value == 19 +// Example: width<4>::value == 2 +// Example: width<10>::value == 1 +// Example: width<1000>::value == 3 +template + struct width +{ + static_assert(d > 0, "width called with zero denominator"); + static CONSTDATA unsigned value = 1 + width::value; +}; + +template +struct width +{ + static CONSTDATA unsigned value = 0; +}; + +template +struct static_pow10 +{ +private: + static CONSTDATA std::uint64_t h = static_pow10::value; +public: + static CONSTDATA std::uint64_t value = h * h * (exp % 2 ? 10 : 1); +}; + +template <> +struct static_pow10<0> +{ + static CONSTDATA std::uint64_t value = 1; +}; + +template +class decimal_format_seconds +{ + using CT = typename std::common_type::type; + using rep = typename CT::rep; + static unsigned CONSTDATA trial_width = + detail::width::value; +public: + static unsigned CONSTDATA width = trial_width < 19 ? trial_width : 6u; + using precision = std::chrono::duration::value>>; + +private: + std::chrono::seconds s_; + precision sub_s_; + +public: + CONSTCD11 decimal_format_seconds() + : s_() + , sub_s_() + {} + + CONSTCD11 explicit decimal_format_seconds(const Duration& d) NOEXCEPT + : s_(std::chrono::duration_cast(d)) + , sub_s_(std::chrono::duration_cast(d - s_)) + {} + + CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT {return s_;} + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_;} + CONSTCD11 precision subseconds() const NOEXCEPT {return sub_s_;} + + CONSTCD14 precision to_duration() const NOEXCEPT + { + return s_ + sub_s_; + } + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + return sub_s_ < std::chrono::seconds{1} && s_ < std::chrono::minutes{1}; + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const decimal_format_seconds& x) + { + return x.print(os, std::chrono::treat_as_floating_point{}); + } + + template + std::basic_ostream& + print(std::basic_ostream& os, std::true_type) const + { + date::detail::save_ostream _(os); + std::chrono::duration d = s_ + sub_s_; + if (d < std::chrono::seconds{10}) + os << '0'; + os << std::fixed << d.count(); + return os; + } + + template + std::basic_ostream& + print(std::basic_ostream& os, std::false_type) const + { + date::detail::save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << s_.count(); + if (width > 0) + { +#if !ONLY_C_LOCALE + os << std::use_facet>(os.getloc()).decimal_point(); +#else + os << '.'; +#endif + date::detail::save_ostream _s(os); + os.imbue(std::locale::classic()); + os.width(width); + os << sub_s_.count(); + } + return os; + } +}; + +template +inline +CONSTCD11 +typename std::enable_if +< + std::numeric_limits::is_signed, + std::chrono::duration +>::type +abs(std::chrono::duration d) +{ + return d >= d.zero() ? +d : -d; +} + +template +inline +CONSTCD11 +typename std::enable_if +< + !std::numeric_limits::is_signed, + std::chrono::duration +>::type +abs(std::chrono::duration d) +{ + return d; +} + +} // namespace detail + +template +class hh_mm_ss +{ + using dfs = detail::decimal_format_seconds::type>; + + std::chrono::hours h_; + std::chrono::minutes m_; + dfs s_; + bool neg_; + +public: + static unsigned CONSTDATA fractional_width = dfs::width; + using precision = typename dfs::precision; + + CONSTCD11 hh_mm_ss() NOEXCEPT + : hh_mm_ss(Duration::zero()) + {} + + CONSTCD11 explicit hh_mm_ss(Duration d) NOEXCEPT + : h_(std::chrono::duration_cast(detail::abs(d))) + , m_(std::chrono::duration_cast(detail::abs(d)) - h_) + , s_(detail::abs(d) - h_ - m_) + , neg_(d < Duration::zero()) + {} + + CONSTCD11 std::chrono::hours hours() const NOEXCEPT {return h_;} + CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT {return m_;} + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_.seconds();} + CONSTCD14 std::chrono::seconds& + seconds(detail::undocumented) NOEXCEPT {return s_.seconds();} + CONSTCD11 precision subseconds() const NOEXCEPT {return s_.subseconds();} + CONSTCD11 bool is_negative() const NOEXCEPT {return neg_;} + + CONSTCD11 explicit operator precision() const NOEXCEPT {return to_duration();} + CONSTCD11 precision to_duration() const NOEXCEPT + {return (s_.to_duration() + m_ + h_) * (1-2*neg_);} + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + return !neg_ && h_ < days{1} && m_ < std::chrono::hours{1} && + s_.in_conventional_range(); + } + +private: + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, hh_mm_ss const& tod) + { + if (tod.is_negative()) + os << '-'; + if (tod.h_ < std::chrono::hours{10}) + os << '0'; + os << tod.h_.count() << ':'; + if (tod.m_ < std::chrono::minutes{10}) + os << '0'; + os << tod.m_.count() << ':' << tod.s_; + return os; + } + + template + friend + std::basic_ostream& + date::to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev, + const std::chrono::seconds* offset_sec); + + template + friend + std::basic_istream& + date::from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, + std::basic_string* abbrev, std::chrono::minutes* offset); +}; + +inline +CONSTCD14 +bool +is_am(std::chrono::hours const& h) NOEXCEPT +{ + using std::chrono::hours; + return hours{0} <= h && h < hours{12}; +} + +inline +CONSTCD14 +bool +is_pm(std::chrono::hours const& h) NOEXCEPT +{ + using std::chrono::hours; + return hours{12} <= h && h < hours{24}; +} + +inline +CONSTCD14 +std::chrono::hours +make12(std::chrono::hours h) NOEXCEPT +{ + using std::chrono::hours; + if (h < hours{12}) + { + if (h == hours{0}) + h = hours{12}; + } + else + { + if (h != hours{12}) + h = h - hours{12}; + } + return h; +} + +inline +CONSTCD14 +std::chrono::hours +make24(std::chrono::hours h, bool is_pm) NOEXCEPT +{ + using std::chrono::hours; + if (is_pm) + { + if (h != hours{12}) + h = h + hours{12}; + } + else if (h == hours{12}) + h = hours{0}; + return h; +} + +template +using time_of_day = hh_mm_ss; + +template ::value>::type> + CONSTCD11 + inline + hh_mm_ss> + make_time(const std::chrono::duration& d) +{ + return hh_mm_ss>(d); +} + +template +inline +typename std::enable_if +< + !std::chrono::treat_as_floating_point::value && + std::ratio_less::value + , std::basic_ostream& +>::type +operator<<(std::basic_ostream& os, const sys_time& tp) +{ + auto const dp = date::floor(tp); + return os << year_month_day(dp) << ' ' << make_time(tp-dp); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const sys_days& dp) +{ + return os << year_month_day(dp); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const local_time& ut) +{ + return (os << sys_time{ut.time_since_epoch()}); +} + +namespace detail +{ + +template +class string_literal; + +template +inline +CONSTCD14 +string_literal::type, + N1 + N2 - 1> + operator+(const string_literal& x, const string_literal& y) NOEXCEPT; + +template +class string_literal +{ + CharT p_[N]; + + CONSTCD11 string_literal() NOEXCEPT + : p_{} + {} + +public: + using const_iterator = const CharT*; + + string_literal(string_literal const&) = default; + string_literal& operator=(string_literal const&) = delete; + + template ::type> + CONSTCD11 string_literal(CharT c) NOEXCEPT + : p_{c} + { + } + + template ::type> + CONSTCD11 string_literal(CharT c1, CharT c2) NOEXCEPT + : p_{c1, c2} + { + } + + template ::type> + CONSTCD11 string_literal(CharT c1, CharT c2, CharT c3) NOEXCEPT + : p_{c1, c2, c3} + { + } + + CONSTCD14 string_literal(const CharT(&a)[N]) NOEXCEPT + : p_{} + { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + template ::type> + CONSTCD14 string_literal(const char(&a)[N]) NOEXCEPT + : p_{} + { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + template ::value>::type> + CONSTCD14 string_literal(string_literal const& a) NOEXCEPT + : p_{} + { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + CONSTCD11 const CharT* data() const NOEXCEPT {return p_;} + CONSTCD11 std::size_t size() const NOEXCEPT {return N-1;} + + CONSTCD11 const_iterator begin() const NOEXCEPT {return p_;} + CONSTCD11 const_iterator end() const NOEXCEPT {return p_ + N-1;} + + CONSTCD11 CharT const& operator[](std::size_t n) const NOEXCEPT + { + return p_[n]; + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const string_literal& s) + { + return os << s.p_; + } + + template + friend + CONSTCD14 + string_literal::type, + N1 + N2 - 1> + operator+(const string_literal& x, const string_literal& y) NOEXCEPT; +}; + +template +CONSTCD11 +inline +string_literal +operator+(const string_literal& x, const string_literal& y) NOEXCEPT +{ + return string_literal(x[0], y[0]); +} + +template +CONSTCD11 +inline +string_literal +operator+(const string_literal& x, const string_literal& y) NOEXCEPT +{ + return string_literal(x[0], x[1], y[0]); +} + +template +CONSTCD14 +inline +string_literal::type, + N1 + N2 - 1> + operator+(const string_literal& x, const string_literal& y) NOEXCEPT +{ + using CT = typename std::conditional::type; + + string_literal r; + std::size_t i = 0; + for (; i < N1-1; ++i) + r.p_[i] = CT(x.p_[i]); + for (std::size_t j = 0; j < N2; ++j, ++i) + r.p_[i] = CT(y.p_[j]); + + return r; +} + + +template +inline +std::basic_string +operator+(std::basic_string x, const string_literal& y) +{ + x.append(y.data(), y.size()); + return x; +} + +#if __cplusplus >= 201402 && (!defined(__EDG_VERSION__) || __EDG_VERSION__ > 411) \ + && (!defined(__SUNPRO_CC) || __SUNPRO_CC > 0x5150) + +template ::value || + std::is_same::value || + std::is_same::value || + std::is_same::value>> + CONSTCD14 + inline + string_literal + msl(CharT c) NOEXCEPT +{ + return string_literal{c}; +} + +CONSTCD14 +inline +std::size_t +to_string_len(std::intmax_t i) +{ + std::size_t r = 0; + do + { + i /= 10; + ++r; + } while (i > 0); + return r; +} + +template +CONSTCD14 +inline +std::enable_if_t +< + N < 10, + string_literal + > + msl() NOEXCEPT +{ + return msl(char(N % 10 + '0')); +} + +template +CONSTCD14 +inline +std::enable_if_t +< + 10 <= N, + string_literal +> +msl() NOEXCEPT +{ + return msl() + msl(char(N % 10 + '0')); +} + +template +CONSTCD14 +inline +std::enable_if_t +< + std::ratio::type::den != 1, + string_literal::type::num) + + to_string_len(std::ratio::type::den) + 4> +> +msl(std::ratio) NOEXCEPT +{ + using R = typename std::ratio::type; + return msl(CharT{'['}) + msl() + msl(CharT{'/'}) + + msl() + msl(CharT{']'}); +} + +template +CONSTCD14 +inline +std::enable_if_t +< + std::ratio::type::den == 1, + string_literal::type::num) + 3> +> +msl(std::ratio) NOEXCEPT +{ + using R = typename std::ratio::type; + return msl(CharT{'['}) + msl() + msl(CharT{']'}); +} + + +#else // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) + +inline +std::string +to_string(std::uint64_t x) +{ + return std::to_string(x); +} + +template +inline +std::basic_string +to_string(std::uint64_t x) +{ + auto y = std::to_string(x); + return std::basic_string(y.begin(), y.end()); +} + +template +inline +typename std::enable_if +< + std::ratio::type::den != 1, + std::basic_string +>::type +msl(std::ratio) +{ + using R = typename std::ratio::type; + return std::basic_string(1, '[') + to_string(R::num) + CharT{'/'} + + to_string(R::den) + CharT{']'}; +} + +template +inline +typename std::enable_if +< + std::ratio::type::den == 1, + std::basic_string +>::type +msl(std::ratio) +{ + using R = typename std::ratio::type; + return std::basic_string(1, '[') + to_string(R::num) + CharT{']'}; +} + +#endif // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) + +template +CONSTCD11 +inline +string_literal +msl(std::atto) NOEXCEPT +{ + return string_literal{'a'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::femto) NOEXCEPT +{ + return string_literal{'f'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::pico) NOEXCEPT +{ + return string_literal{'p'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::nano) NOEXCEPT +{ + return string_literal{'n'}; +} + +template +CONSTCD11 +inline +typename std::enable_if +< + std::is_same::value, + string_literal +>::type +msl(std::micro) NOEXCEPT +{ + return string_literal{'\xC2', '\xB5'}; +} + +template +CONSTCD11 +inline +typename std::enable_if +< + !std::is_same::value, + string_literal +>::type +msl(std::micro) NOEXCEPT +{ + return string_literal{CharT{static_cast('\xB5')}}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::milli) NOEXCEPT +{ + return string_literal{'m'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::centi) NOEXCEPT +{ + return string_literal{'c'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::deca) NOEXCEPT +{ + return string_literal{'d', 'a'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::deci) NOEXCEPT +{ + return string_literal{'d'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::hecto) NOEXCEPT +{ + return string_literal{'h'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::kilo) NOEXCEPT +{ + return string_literal{'k'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::mega) NOEXCEPT +{ + return string_literal{'M'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::giga) NOEXCEPT +{ + return string_literal{'G'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::tera) NOEXCEPT +{ + return string_literal{'T'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::peta) NOEXCEPT +{ + return string_literal{'P'}; +} + +template +CONSTCD11 +inline +string_literal +msl(std::exa) NOEXCEPT +{ + return string_literal{'E'}; +} + +template +CONSTCD11 +inline +auto +get_units(Period p) +-> decltype(msl(p) + string_literal{'s'}) +{ + return msl(p) + string_literal{'s'}; +} + +template +CONSTCD11 +inline +string_literal +get_units(std::ratio<1>) +{ + return string_literal{'s'}; +} + +template +CONSTCD11 +inline +string_literal +get_units(std::ratio<3600>) +{ + return string_literal{'h'}; +} + +template +CONSTCD11 +inline +string_literal +get_units(std::ratio<60>) +{ + return string_literal{'m', 'i', 'n'}; +} + +template +CONSTCD11 +inline +string_literal +get_units(std::ratio<86400>) +{ + return string_literal{'d'}; +} + +template > +struct make_string; + +template <> +struct make_string +{ + template + static + std::string + from(Rep n) + { + return std::to_string(n); + } +}; + +template +struct make_string +{ + template + static + std::basic_string + from(Rep n) + { + auto s = std::to_string(n); + return std::basic_string(s.begin(), s.end()); + } +}; + +template <> +struct make_string +{ + template + static + std::wstring + from(Rep n) + { + return std::to_wstring(n); + } +}; + +template +struct make_string +{ + template + static + std::basic_string + from(Rep n) + { + auto s = std::to_wstring(n); + return std::basic_string(s.begin(), s.end()); + } +}; + +} // namespace detail + + // to_stream + +CONSTDATA year nanyear{-32768}; + +template +struct fields +{ + year_month_day ymd{nanyear/0/0}; + weekday wd{8u}; + hh_mm_ss tod{}; + bool has_tod = false; + + fields() = default; + + fields(year_month_day ymd_) : ymd(ymd_) {} + fields(weekday wd_) : wd(wd_) {} + fields(hh_mm_ss tod_) : tod(tod_), has_tod(true) {} + + fields(year_month_day ymd_, weekday wd_) : ymd(ymd_), wd(wd_) {} + fields(year_month_day ymd_, hh_mm_ss tod_) : ymd(ymd_), tod(tod_), + has_tod(true) {} + + fields(weekday wd_, hh_mm_ss tod_) : wd(wd_), tod(tod_), has_tod(true) {} + + fields(year_month_day ymd_, weekday wd_, hh_mm_ss tod_) + : ymd(ymd_) + , wd(wd_) + , tod(tod_) + , has_tod(true) + {} +}; + +namespace detail +{ + +template +unsigned +extract_weekday(std::basic_ostream& os, const fields& fds) +{ + if (!fds.ymd.ok() && !fds.wd.ok()) + { + // fds does not contain a valid weekday + os.setstate(std::ios::failbit); + return 8; + } + weekday wd; + if (fds.ymd.ok()) + { + wd = weekday{sys_days(fds.ymd)}; + if (fds.wd.ok() && wd != fds.wd) + { + // fds.ymd and fds.wd are inconsistent + os.setstate(std::ios::failbit); + return 8; + } + } + else + wd = fds.wd; + return static_cast((wd - Sunday).count()); +} + +template +unsigned +extract_month(std::basic_ostream& os, const fields& fds) +{ + if (!fds.ymd.month().ok()) + { + // fds does not contain a valid month + os.setstate(std::ios::failbit); + return 0; + } + return static_cast(fds.ymd.month()); +} + +} // namespace detail + +#if ONLY_C_LOCALE + +namespace detail +{ + +inline +std::pair +weekday_names() +{ + static const std::string nm[] = + { + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + "Sun", + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat" + }; + return std::make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); +} + +inline +std::pair +month_names() +{ + static const std::string nm[] = + { + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec" + }; + return std::make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); +} + +inline +std::pair +ampm_names() +{ + static const std::string nm[] = + { + "AM", + "PM" + }; + return std::make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); +} + +template +FwdIter +scan_keyword(std::basic_istream& is, FwdIter kb, FwdIter ke) +{ + size_t nkw = static_cast(std::distance(kb, ke)); + const unsigned char doesnt_match = '\0'; + const unsigned char might_match = '\1'; + const unsigned char does_match = '\2'; + unsigned char statbuf[100]; + unsigned char* status = statbuf; + std::unique_ptr stat_hold(0, free); + if (nkw > sizeof(statbuf)) + { + status = (unsigned char*)std::malloc(nkw); + if (status == nullptr) + throw std::bad_alloc(); + stat_hold.reset(status); + } + size_t n_might_match = nkw; // At this point, any keyword might match + size_t n_does_match = 0; // but none of them definitely do + // Initialize all statuses to might_match, except for "" keywords are does_match + unsigned char* st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) + { + if (!ky->empty()) + *st = might_match; + else + { + *st = does_match; + --n_might_match; + ++n_does_match; + } + } + // While there might be a match, test keywords against the next CharT + for (size_t indx = 0; is && n_might_match > 0; ++indx) + { + // Peek at the next CharT but don't consume it + auto ic = is.peek(); + if (ic == EOF) + { + is.setstate(std::ios::eofbit); + break; + } + auto c = static_cast(toupper(ic)); + bool consume = false; + // For each keyword which might match, see if the indx character is c + // If a match if found, consume c + // If a match is found, and that is the last character in the keyword, + // then that keyword matches. + // If the keyword doesn't match this character, then change the keyword + // to doesn't match + st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) + { + if (*st == might_match) + { + if (c == static_cast(toupper((*ky)[indx]))) + { + consume = true; + if (ky->size() == indx+1) + { + *st = does_match; + --n_might_match; + ++n_does_match; + } + } + else + { + *st = doesnt_match; + --n_might_match; + } + } + } + // consume if we matched a character + if (consume) + { + (void)is.get(); + // If we consumed a character and there might be a matched keyword that + // was marked matched on a previous iteration, then such keywords + // are now marked as not matching. + if (n_might_match + n_does_match > 1) + { + st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) + { + if (*st == does_match && ky->size() != indx+1) + { + *st = doesnt_match; + --n_does_match; + } + } + } + } + } + // We've exited the loop because we hit eof and/or we have no more "might matches". + // Return the first matching result + for (st = status; kb != ke; ++kb, ++st) + if (*st == does_match) + break; + if (kb == ke) + is.setstate(std::ios::failbit); + return kb; +} + +} // namespace detail + +#endif // ONLY_C_LOCALE + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev, + const std::chrono::seconds* offset_sec) +{ +#if ONLY_C_LOCALE + using detail::weekday_names; + using detail::month_names; + using detail::ampm_names; +#endif + using detail::save_ostream; + using detail::get_units; + using detail::extract_weekday; + using detail::extract_month; + using std::ios; + using std::chrono::duration_cast; + using std::chrono::seconds; + using std::chrono::minutes; + using std::chrono::hours; + date::detail::save_ostream ss(os); + os.fill(' '); + os.flags(std::ios::skipws | std::ios::dec); + os.width(0); + tm tm{}; + bool insert_negative = fds.has_tod && fds.tod.to_duration() < Duration::zero(); +#if !ONLY_C_LOCALE + auto& facet = std::use_facet>(os.getloc()); +#endif + const CharT* command = nullptr; + CharT modified = CharT{}; + for (; *fmt; ++fmt) + { + switch (*fmt) + { + case 'a': + case 'A': + if (command) + { + if (modified == CharT{}) + { + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; +#if !ONLY_C_LOCALE + const CharT f[] = {'%', *fmt}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); +#else // ONLY_C_LOCALE + os << weekday_names().first[tm.tm_wday+7*(*fmt == 'a')]; +#endif // ONLY_C_LOCALE + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'b': + case 'B': + case 'h': + if (command) + { + if (modified == CharT{}) + { + tm.tm_mon = static_cast(extract_month(os, fds)) - 1; +#if !ONLY_C_LOCALE + const CharT f[] = {'%', *fmt}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); +#else // ONLY_C_LOCALE + os << month_names().first[tm.tm_mon+12*(*fmt != 'B')]; +#endif // ONLY_C_LOCALE + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'c': + case 'x': + if (command) + { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + if (*fmt == 'c' && !fds.has_tod) + os.setstate(std::ios::failbit); +#if !ONLY_C_LOCALE + tm = std::tm{}; + auto const& ymd = fds.ymd; + auto ld = local_days(ymd); + if (*fmt == 'c') + { + tm.tm_sec = static_cast(fds.tod.seconds().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_hour = static_cast(fds.tod.hours().count()); + } + tm.tm_mday = static_cast(static_cast(ymd.day())); + tm.tm_mon = static_cast(extract_month(os, fds) - 1); + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + CharT f[3] = {'%'}; + auto fe = std::begin(f) + 1; + if (modified == CharT{'E'}) + *fe++ = modified; + *fe++ = *fmt; + facet.put(os, os, os.fill(), &tm, std::begin(f), fe); +#else // ONLY_C_LOCALE + if (*fmt == 'c') + { + auto wd = static_cast(extract_weekday(os, fds)); + os << weekday_names().first[static_cast(wd)+7] + << ' '; + os << month_names().first[extract_month(os, fds)-1+12] << ' '; + auto d = static_cast(static_cast(fds.ymd.day())); + if (d < 10) + os << ' '; + os << d << ' ' + << make_time(duration_cast(fds.tod.to_duration())) + << ' ' << fds.ymd.year(); + + } + else // *fmt == 'x' + { + auto const& ymd = fds.ymd; + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(ymd.month()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.day()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.year()) % 100; + } +#endif // ONLY_C_LOCALE + } + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'C': + if (command) + { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.ymd.year().ok()) + os.setstate(std::ios::failbit); + auto y = static_cast(fds.ymd.year()); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + if (y >= 0) + { + os.width(2); + os << y/100; + } + else + { + os << CharT{'-'}; + os.width(2); + os << -(y-99)/100; + } + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'E'}) + { + tm.tm_year = y - 1900; + CharT f[3] = {'%', 'E', 'C'}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'd': + case 'e': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.ymd.day().ok()) + os.setstate(std::ios::failbit); + auto d = static_cast(static_cast(fds.ymd.day())); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + save_ostream _(os); + if (*fmt == CharT{'d'}) + os.fill('0'); + else + os.fill(' '); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << d; + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + tm.tm_mday = d; + CharT f[3] = {'%', 'O', *fmt}; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'D': + if (command) + { + if (modified == CharT{}) + { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto const& ymd = fds.ymd; + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(ymd.month()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.day()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.year()) % 100; + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'F': + if (command) + { + if (modified == CharT{}) + { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto const& ymd = fds.ymd; + save_ostream _(os); + os.imbue(std::locale::classic()); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(4); + os << static_cast(ymd.year()) << CharT{'-'}; + os.width(2); + os << static_cast(ymd.month()) << CharT{'-'}; + os.width(2); + os << static_cast(ymd.day()); + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'g': + case 'G': + if (command) + { + if (modified == CharT{}) + { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(fds.ymd); + auto y = year_month_day{ld + days{3}}.year(); + auto start = local_days((y-years{1})/December/Thursday[last]) + + (Monday-Thursday); + if (ld < start) + --y; + if (*fmt == CharT{'G'}) + os << y; + else + { + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << std::abs(static_cast(y)) % 100; + } + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'H': + case 'I': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (insert_negative) + { + os << '-'; + insert_negative = false; + } + auto hms = fds.tod; +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + auto h = *fmt == CharT{'I'} ? date::make12(hms.hours()) : hms.hours(); + if (h < hours{10}) + os << CharT{'0'}; + os << h.count(); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_hour = static_cast(hms.hours().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'j': + if (command) + { + if (modified == CharT{}) + { + if (fds.ymd.ok() || fds.has_tod) + { + days doy; + if (fds.ymd.ok()) + { + auto ld = local_days(fds.ymd); + auto y = fds.ymd.year(); + doy = ld - local_days(y/January/1) + days{1}; + } + else + { + doy = duration_cast(fds.tod.to_duration()); + } + save_ostream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(3); + os << doy.count(); + } + else + { + os.setstate(std::ios::failbit); + } + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'm': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.ymd.month().ok()) + os.setstate(std::ios::failbit); + auto m = static_cast(fds.ymd.month()); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + if (m < 10) + os << CharT{'0'}; + os << m; + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_mon = static_cast(m-1); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'M': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (insert_negative) + { + os << '-'; + insert_negative = false; + } +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + if (fds.tod.minutes() < minutes{10}) + os << CharT{'0'}; + os << fds.tod.minutes().count(); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_min = static_cast(fds.tod.minutes().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'n': + if (command) + { + if (modified == CharT{}) + os << CharT{'\n'}; + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'p': + if (command) + { + if (modified == CharT{}) + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); +#if !ONLY_C_LOCALE + const CharT f[] = {'%', *fmt}; + tm.tm_hour = static_cast(fds.tod.hours().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); +#else + if (date::is_am(fds.tod.hours())) + os << ampm_names().first[0]; + else + os << ampm_names().first[1]; +#endif + } + else + { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'Q': + case 'q': + if (command) + { + if (modified == CharT{}) + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + auto d = fds.tod.to_duration(); + if (*fmt == 'q') + os << get_units(typename decltype(d)::period::type{}); + else + os << d.count(); + } + else + { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'r': + if (command) + { + if (modified == CharT{}) + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); +#if !ONLY_C_LOCALE + const CharT f[] = {'%', *fmt}; + tm.tm_hour = static_cast(fds.tod.hours().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_sec = static_cast(fds.tod.seconds().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); +#else + hh_mm_ss tod(duration_cast(fds.tod.to_duration())); + save_ostream _(os); + os.fill('0'); + os.width(2); + os << date::make12(tod.hours()).count() << CharT{':'}; + os.width(2); + os << tod.minutes().count() << CharT{':'}; + os.width(2); + os << tod.seconds().count() << CharT{' '}; + if (date::is_am(tod.hours())) + os << ampm_names().first[0]; + else + os << ampm_names().first[1]; +#endif + } + else + { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'R': + if (command) + { + if (modified == CharT{}) + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (fds.tod.hours() < hours{10}) + os << CharT{'0'}; + os << fds.tod.hours().count() << CharT{':'}; + if (fds.tod.minutes() < minutes{10}) + os << CharT{'0'}; + os << fds.tod.minutes().count(); + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'S': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + if (insert_negative) + { + os << '-'; + insert_negative = false; + } +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + os << fds.tod.s_; + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_sec = static_cast(fds.tod.s_.seconds().count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 't': + if (command) + { + if (modified == CharT{}) + os << CharT{'\t'}; + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'T': + if (command) + { + if (modified == CharT{}) + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); + os << fds.tod; + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'u': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + auto wd = extract_weekday(os, fds); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + os << (wd != 0 ? wd : 7u); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_wday = static_cast(wd); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'U': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + auto const& ymd = fds.ymd; + if (!ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(ymd); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + auto st = local_days(Sunday[1]/January/ymd.year()); + if (ld < st) + os << CharT{'0'} << CharT{'0'}; + else + { + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'V': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(fds.ymd); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + auto y = year_month_day{ld + days{3}}.year(); + auto st = local_days((y-years{1})/12/Thursday[last]) + + (Monday-Thursday); + if (ld < st) + { + --y; + st = local_days((y - years{1})/12/Thursday[last]) + + (Monday-Thursday); + } + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + auto const& ymd = fds.ymd; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'w': + if (command) + { + auto wd = extract_weekday(os, fds); + if (os.fail()) + return os; +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) +#endif + { + os << wd; + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_wday = static_cast(wd); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + else + { + os << CharT{'%'} << modified << *fmt; + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'W': + if (command) + { + if (modified == CharT{'E'}) + os << CharT{'%'} << modified << *fmt; + else + { + auto const& ymd = fds.ymd; + if (!ymd.ok()) + os.setstate(std::ios::failbit); + auto ld = local_days(ymd); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + auto st = local_days(Monday[1]/January/ymd.year()); + if (ld < st) + os << CharT{'0'} << CharT{'0'}; + else + { + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'X': + if (command) + { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.has_tod) + os.setstate(std::ios::failbit); +#if !ONLY_C_LOCALE + tm = std::tm{}; + tm.tm_sec = static_cast(fds.tod.seconds().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_hour = static_cast(fds.tod.hours().count()); + CharT f[3] = {'%'}; + auto fe = std::begin(f) + 1; + if (modified == CharT{'E'}) + *fe++ = modified; + *fe++ = *fmt; + facet.put(os, os, os.fill(), &tm, std::begin(f), fe); +#else + os << fds.tod; +#endif + } + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'y': + if (command) + { + if (!fds.ymd.year().ok()) + os.setstate(std::ios::failbit); + auto y = static_cast(fds.ymd.year()); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + y = std::abs(y) % 100; + if (y < 10) + os << CharT{'0'}; + os << y; +#if !ONLY_C_LOCALE + } + else + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = y - 1900; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'Y': + if (command) + { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else + { + if (!fds.ymd.year().ok()) + os.setstate(std::ios::failbit); + auto y = fds.ymd.year(); +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + save_ostream _(os); + os.imbue(std::locale::classic()); + os << y; + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'E'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(y) - 1900; + facet.put(os, os, os.fill(), &tm, std::begin(f), std::end(f)); + } +#endif + } + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'z': + if (command) + { + if (offset_sec == nullptr) + { + // Can not format %z with unknown offset + os.setstate(ios::failbit); + return os; + } + auto m = duration_cast(*offset_sec); + auto neg = m < minutes{0}; + m = date::abs(m); + auto h = duration_cast(m); + m -= h; + if (neg) + os << CharT{'-'}; + else + os << CharT{'+'}; + if (h < hours{10}) + os << CharT{'0'}; + os << h.count(); + if (modified != CharT{}) + os << CharT{':'}; + if (m < minutes{10}) + os << CharT{'0'}; + os << m.count(); + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'Z': + if (command) + { + if (modified == CharT{}) + { + if (abbrev == nullptr) + { + // Can not format %Z with unknown time_zone + os.setstate(ios::failbit); + return os; + } + for (auto c : *abbrev) + os << CharT(c); + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'E': + case 'O': + if (command) + { + if (modified == CharT{}) + { + modified = *fmt; + } + else + { + os << CharT{'%'} << modified << *fmt; + command = nullptr; + modified = CharT{}; + } + } + else + os << *fmt; + break; + case '%': + if (command) + { + if (modified == CharT{}) + { + os << CharT{'%'}; + command = nullptr; + } + else + { + os << CharT{'%'} << modified << CharT{'%'}; + command = nullptr; + modified = CharT{}; + } + } + else + command = fmt; + break; + default: + if (command) + { + os << CharT{'%'}; + command = nullptr; + } + if (modified != CharT{}) + { + os << modified; + modified = CharT{}; + } + os << *fmt; + break; + } + } + if (command) + os << CharT{'%'}; + if (modified != CharT{}) + os << modified; + return os; +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const year& y) +{ + using CT = std::chrono::seconds; + fields fds{y/0/0}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const month& m) +{ + using CT = std::chrono::seconds; + fields fds{m/0/nanyear}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const day& d) +{ + using CT = std::chrono::seconds; + fields fds{d/0/nanyear}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const weekday& wd) +{ + using CT = std::chrono::seconds; + fields fds{wd}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const year_month& ym) +{ + using CT = std::chrono::seconds; + fields fds{ym/0}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const month_day& md) +{ + using CT = std::chrono::seconds; + fields fds{md/nanyear}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const year_month_day& ymd) +{ + using CT = std::chrono::seconds; + fields fds{ymd}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const std::chrono::duration& d) +{ + using Duration = std::chrono::duration; + using CT = typename std::common_type::type; + fields fds{hh_mm_ss{d}}; + return to_stream(os, fmt, fds); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const local_time& tp, const std::string* abbrev = nullptr, + const std::chrono::seconds* offset_sec = nullptr) +{ + using CT = typename std::common_type::type; + auto ld = floor(tp); + fields fds{year_month_day{ld}, hh_mm_ss{tp-local_seconds{ld}}}; + return to_stream(os, fmt, fds, abbrev, offset_sec); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const sys_time& tp) +{ + using std::chrono::seconds; + using CT = typename std::common_type::type; + const std::string abbrev("UTC"); + CONSTDATA seconds offset{0}; + auto sd = floor(tp); + fields fds{year_month_day{sd}, hh_mm_ss{tp-sys_seconds{sd}}}; + return to_stream(os, fmt, fds, &abbrev, &offset); +} + +// format + +template +auto +format(const std::locale& loc, const CharT* fmt, const Streamable& tp) +-> decltype(to_stream(std::declval&>(), fmt, tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + os.imbue(loc); + to_stream(os, fmt, tp); + return os.str(); +} + +template +auto +format(const CharT* fmt, const Streamable& tp) +-> decltype(to_stream(std::declval&>(), fmt, tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + to_stream(os, fmt, tp); + return os.str(); +} + +template +auto +format(const std::locale& loc, const std::basic_string& fmt, + const Streamable& tp) + -> decltype(to_stream(std::declval&>(), fmt.c_str(), tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + os.imbue(loc); + to_stream(os, fmt.c_str(), tp); + return os.str(); +} + +template +auto +format(const std::basic_string& fmt, const Streamable& tp) +-> decltype(to_stream(std::declval&>(), fmt.c_str(), tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + to_stream(os, fmt.c_str(), tp); + return os.str(); +} + +// parse + +namespace detail +{ + +template +bool +read_char(std::basic_istream& is, CharT fmt, std::ios::iostate& err) +{ + auto ic = is.get(); + if (Traits::eq_int_type(ic, Traits::eof()) || + !Traits::eq(Traits::to_char_type(ic), fmt)) + { + err |= std::ios::failbit; + is.setstate(std::ios::failbit); + return false; + } + return true; +} + +template +unsigned +read_unsigned(std::basic_istream& is, unsigned m = 1, unsigned M = 10) +{ + unsigned x = 0; + unsigned count = 0; + while (true) + { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + break; + auto c = static_cast(Traits::to_char_type(ic)); + if (!('0' <= c && c <= '9')) + break; + (void)is.get(); + ++count; + x = 10*x + static_cast(c - '0'); + if (count == M) + break; + } + if (count < m) + is.setstate(std::ios::failbit); + return x; +} + +template +int +read_signed(std::basic_istream& is, unsigned m = 1, unsigned M = 10) +{ + auto ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) + { + auto c = static_cast(Traits::to_char_type(ic)); + if (('0' <= c && c <= '9') || c == '-' || c == '+') + { + if (c == '-' || c == '+') + (void)is.get(); + auto x = static_cast(read_unsigned(is, std::max(m, 1u), M)); + if (!is.fail()) + { + if (c == '-') + x = -x; + return x; + } + } + } + if (m > 0) + is.setstate(std::ios::failbit); + return 0; +} + +template +long double +read_long_double(std::basic_istream& is, unsigned m = 1, unsigned M = 10) +{ + unsigned count = 0; + unsigned fcount = 0; + unsigned long long i = 0; + unsigned long long f = 0; + bool parsing_fraction = false; +#if ONLY_C_LOCALE + typename Traits::int_type decimal_point = '.'; +#else + auto decimal_point = Traits::to_int_type( + std::use_facet>(is.getloc()).decimal_point()); +#endif + while (true) + { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + break; + if (Traits::eq_int_type(ic, decimal_point)) + { + decimal_point = Traits::eof(); + parsing_fraction = true; + } + else + { + auto c = static_cast(Traits::to_char_type(ic)); + if (!('0' <= c && c <= '9')) + break; + if (!parsing_fraction) + { + i = 10*i + static_cast(c - '0'); + } + else + { + f = 10*f + static_cast(c - '0'); + ++fcount; + } + } + (void)is.get(); + if (++count == M) + break; + } + if (count < m) + { + is.setstate(std::ios::failbit); + return 0; + } + return i + f/std::pow(10.L, fcount); +} + +struct rs +{ + int& i; + unsigned m; + unsigned M; +}; + +struct ru +{ + int& i; + unsigned m; + unsigned M; +}; + +struct rld +{ + long double& i; + unsigned m; + unsigned M; +}; + +template +void +read(std::basic_istream&) +{ +} + +template +void +read(std::basic_istream& is, CharT a0, Args&& ...args); + +template +void +read(std::basic_istream& is, rs a0, Args&& ...args); + +template +void +read(std::basic_istream& is, ru a0, Args&& ...args); + +template +void +read(std::basic_istream& is, int a0, Args&& ...args); + +template +void +read(std::basic_istream& is, rld a0, Args&& ...args); + +template +void +read(std::basic_istream& is, CharT a0, Args&& ...args) +{ + // No-op if a0 == CharT{} + if (a0 != CharT{}) + { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + { + is.setstate(std::ios::failbit | std::ios::eofbit); + return; + } + if (!Traits::eq(Traits::to_char_type(ic), a0)) + { + is.setstate(std::ios::failbit); + return; + } + (void)is.get(); + } + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, rs a0, Args&& ...args) +{ + auto x = read_signed(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = x; + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, ru a0, Args&& ...args) +{ + auto x = read_unsigned(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = static_cast(x); + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, int a0, Args&& ...args) +{ + if (a0 != -1) + { + auto u = static_cast(a0); + CharT buf[std::numeric_limits::digits10+2u] = {}; + auto e = buf; + do + { + *e++ = static_cast(CharT(u % 10) + CharT{'0'}); + u /= 10; + } while (u > 0); + std::reverse(buf, e); + for (auto p = buf; p != e && is.rdstate() == std::ios::goodbit; ++p) + read(is, *p); + } + if (is.rdstate() == std::ios::goodbit) + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, rld a0, Args&& ...args) +{ + auto x = read_long_double(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = x; + read(is, std::forward(args)...); +} + +template +inline +void +checked_set(T& value, T from, T not_a_value, std::basic_ios& is) +{ + if (!is.fail()) + { + if (value == not_a_value) + value = std::move(from); + else if (value != from) + is.setstate(std::ios::failbit); + } +} + +} // namespace detail; + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, std::basic_string* abbrev, + std::chrono::minutes* offset) +{ + using std::numeric_limits; + using std::ios; + using std::chrono::duration; + using std::chrono::duration_cast; + using std::chrono::seconds; + using std::chrono::minutes; + using std::chrono::hours; + using detail::round_i; + typename std::basic_istream::sentry ok{is, true}; + if (ok) + { + date::detail::save_istream ss(is); + is.fill(' '); + is.flags(std::ios::skipws | std::ios::dec); + is.width(0); +#if !ONLY_C_LOCALE + auto& f = std::use_facet>(is.getloc()); + std::tm tm{}; +#endif + const CharT* command = nullptr; + auto modified = CharT{}; + auto width = -1; + + CONSTDATA int not_a_year = numeric_limits::min(); + CONSTDATA int not_a_2digit_year = 100; + CONSTDATA int not_a_century = not_a_year / 100; + CONSTDATA int not_a_month = 0; + CONSTDATA int not_a_day = 0; + CONSTDATA int not_a_hour = numeric_limits::min(); + CONSTDATA int not_a_hour_12_value = 0; + CONSTDATA int not_a_minute = not_a_hour; + CONSTDATA Duration not_a_second = Duration::min(); + CONSTDATA int not_a_doy = -1; + CONSTDATA int not_a_weekday = 8; + CONSTDATA int not_a_week_num = 100; + CONSTDATA int not_a_ampm = -1; + CONSTDATA minutes not_a_offset = minutes::min(); + + int Y = not_a_year; // c, F, Y * + int y = not_a_2digit_year; // D, x, y * + int g = not_a_2digit_year; // g * + int G = not_a_year; // G * + int C = not_a_century; // C * + int m = not_a_month; // b, B, h, m, c, D, F, x * + int d = not_a_day; // c, d, D, e, F, x * + int j = not_a_doy; // j * + int wd = not_a_weekday; // a, A, u, w * + int H = not_a_hour; // c, H, R, T, X * + int I = not_a_hour_12_value; // I, r * + int p = not_a_ampm; // p, r * + int M = not_a_minute; // c, M, r, R, T, X * + Duration s = not_a_second; // c, r, S, T, X * + int U = not_a_week_num; // U * + int V = not_a_week_num; // V * + int W = not_a_week_num; // W * + std::basic_string temp_abbrev; // Z * + minutes temp_offset = not_a_offset; // z * + + using detail::read; + using detail::rs; + using detail::ru; + using detail::rld; + using detail::checked_set; + for (; *fmt != CharT{} && !is.fail(); ++fmt) + { + switch (*fmt) + { + case 'a': + case 'A': + case 'u': + case 'w': // wd: a, A, u, w + if (command) + { + int trial_wd = not_a_weekday; + if (*fmt == 'a' || *fmt == 'A') + { + if (modified == CharT{}) + { +#if !ONLY_C_LOCALE + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + is.setstate(err); + if (!is.fail()) + trial_wd = tm.tm_wday; +#else + auto nm = detail::weekday_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (!is.fail()) + trial_wd = i % 7; +#endif + } + else + read(is, CharT{'%'}, width, modified, *fmt); + } + else // *fmt == 'u' || *fmt == 'w' + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) +#endif + { + read(is, ru{trial_wd, 1, width == -1 ? + 1u : static_cast(width)}); + if (!is.fail()) + { + if (*fmt == 'u') + { + if (!(1 <= trial_wd && trial_wd <= 7)) + { + trial_wd = not_a_weekday; + is.setstate(ios::failbit); + } + else if (trial_wd == 7) + trial_wd = 0; + } + else // *fmt == 'w' + { + if (!(0 <= trial_wd && trial_wd <= 6)) + { + trial_wd = not_a_weekday; + is.setstate(ios::failbit); + } + } + } + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + is.setstate(err); + if (!is.fail()) + trial_wd = tm.tm_wday; + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + } + if (trial_wd != not_a_weekday) + checked_set(wd, trial_wd, not_a_weekday, is); + } + else // !command + read(is, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + break; + case 'b': + case 'B': + case 'h': + if (command) + { + if (modified == CharT{}) + { + int ttm = not_a_month; +#if !ONLY_C_LOCALE + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + ttm = tm.tm_mon + 1; + is.setstate(err); +#else + auto nm = detail::month_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (!is.fail()) + ttm = i % 12 + 1; +#endif + checked_set(m, ttm, not_a_month, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'c': + if (command) + { + if (modified != CharT{'O'}) + { +#if !ONLY_C_LOCALE + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + checked_set(m, tm.tm_mon + 1, not_a_month, is); + checked_set(d, tm.tm_mday, not_a_day, is); + checked_set(H, tm.tm_hour, not_a_hour, is); + checked_set(M, tm.tm_min, not_a_minute, is); + checked_set(s, duration_cast(seconds{tm.tm_sec}), + not_a_second, is); + } + is.setstate(err); +#else + // "%a %b %e %T %Y" + auto nm = detail::weekday_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + checked_set(wd, static_cast(i % 7), not_a_weekday, is); + ws(is); + nm = detail::month_names(); + i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + checked_set(m, static_cast(i % 12 + 1), not_a_month, is); + ws(is); + int td = not_a_day; + read(is, rs{td, 1, 2}); + checked_set(d, td, not_a_day, is); + ws(is); + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int tH; + int tM; + long double S; + read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, + CharT{':'}, rld{S, 1, w}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), + not_a_second, is); + ws(is); + int tY = not_a_year; + read(is, rs{tY, 1, 4u}); + checked_set(Y, tY, not_a_year, is); +#endif + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'x': + if (command) + { + if (modified != CharT{'O'}) + { +#if !ONLY_C_LOCALE + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + checked_set(m, tm.tm_mon + 1, not_a_month, is); + checked_set(d, tm.tm_mday, not_a_day, is); + } + is.setstate(err); +#else + // "%m/%d/%y" + int ty = not_a_2digit_year; + int tm = not_a_month; + int td = not_a_day; + read(is, ru{tm, 1, 2}, CharT{'/'}, ru{td, 1, 2}, CharT{'/'}, + rs{ty, 1, 2}); + checked_set(y, ty, not_a_2digit_year, is); + checked_set(m, tm, not_a_month, is); + checked_set(d, td, not_a_day, is); +#endif + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'X': + if (command) + { + if (modified != CharT{'O'}) + { +#if !ONLY_C_LOCALE + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + checked_set(H, tm.tm_hour, not_a_hour, is); + checked_set(M, tm.tm_min, not_a_minute, is); + checked_set(s, duration_cast(seconds{tm.tm_sec}), + not_a_second, is); + } + is.setstate(err); +#else + // "%T" + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int tH = not_a_hour; + int tM = not_a_minute; + long double S; + read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, + CharT{':'}, rld{S, 1, w}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), + not_a_second, is); +#endif + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'C': + if (command) + { + int tC = not_a_century; +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + read(is, rs{tC, 1, width == -1 ? 2u : static_cast(width)}); +#if !ONLY_C_LOCALE + } + else + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + auto tY = tm.tm_year + 1900; + tC = (tY >= 0 ? tY : tY-99) / 100; + } + is.setstate(err); + } +#endif + checked_set(C, tC, not_a_century, is); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'D': + if (command) + { + if (modified == CharT{}) + { + int tn = not_a_month; + int td = not_a_day; + int ty = not_a_2digit_year; + read(is, ru{tn, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, + ru{td, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, + rs{ty, 1, 2}); + checked_set(y, ty, not_a_2digit_year, is); + checked_set(m, tn, not_a_month, is); + checked_set(d, td, not_a_day, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'F': + if (command) + { + if (modified == CharT{}) + { + int tY = not_a_year; + int tn = not_a_month; + int td = not_a_day; + read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}, + CharT{'-'}, ru{tn, 1, 2}, CharT{'-'}, ru{td, 1, 2}); + checked_set(Y, tY, not_a_year, is); + checked_set(m, tn, not_a_month, is); + checked_set(d, td, not_a_day, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'd': + case 'e': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) +#endif + { + int td = not_a_day; + read(is, rs{td, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(d, td, not_a_day, is); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + command = nullptr; + width = -1; + modified = CharT{}; + if ((err & ios::failbit) == 0) + checked_set(d, tm.tm_mday, not_a_day, is); + is.setstate(err); + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'H': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) +#endif + { + int tH = not_a_hour; + read(is, ru{tH, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(H, tH, not_a_hour, is); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + checked_set(H, tm.tm_hour, not_a_hour, is); + is.setstate(err); + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'I': + if (command) + { + if (modified == CharT{}) + { + int tI = not_a_hour_12_value; + // reads in an hour into I, but most be in [1, 12] + read(is, rs{tI, 1, width == -1 ? 2u : static_cast(width)}); + if (!(1 <= tI && tI <= 12)) + is.setstate(ios::failbit); + checked_set(I, tI, not_a_hour_12_value, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'j': + if (command) + { + if (modified == CharT{}) + { + int tj = not_a_doy; + read(is, ru{tj, 1, width == -1 ? 3u : static_cast(width)}); + checked_set(j, tj, not_a_doy, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'M': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) +#endif + { + int tM = not_a_minute; + read(is, ru{tM, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(M, tM, not_a_minute, is); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + checked_set(M, tm.tm_min, not_a_minute, is); + is.setstate(err); + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'm': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) +#endif + { + int tn = not_a_month; + read(is, rs{tn, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(m, tn, not_a_month, is); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + checked_set(m, tm.tm_mon + 1, not_a_month, is); + is.setstate(err); + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'n': + case 't': + if (command) + { + if (modified == CharT{}) + { + // %n matches a single white space character + // %t matches 0 or 1 white space characters + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + { + ios::iostate err = ios::eofbit; + if (*fmt == 'n') + err |= ios::failbit; + is.setstate(err); + break; + } + if (isspace(ic)) + { + (void)is.get(); + } + else if (*fmt == 'n') + is.setstate(ios::failbit); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'p': + if (command) + { + if (modified == CharT{}) + { + int tp = not_a_ampm; +#if !ONLY_C_LOCALE + tm = std::tm{}; + tm.tm_hour = 1; + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + is.setstate(err); + if (tm.tm_hour == 1) + tp = 0; + else if (tm.tm_hour == 13) + tp = 1; + else + is.setstate(err); +#else + auto nm = detail::ampm_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + tp = static_cast(i); +#endif + checked_set(p, tp, not_a_ampm, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + + break; + case 'r': + if (command) + { + if (modified == CharT{}) + { +#if !ONLY_C_LOCALE + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + checked_set(H, tm.tm_hour, not_a_hour, is); + checked_set(M, tm.tm_min, not_a_hour, is); + checked_set(s, duration_cast(seconds{tm.tm_sec}), + not_a_second, is); + } + is.setstate(err); +#else + // "%I:%M:%S %p" + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + long double S; + int tI = not_a_hour_12_value; + int tM = not_a_minute; + read(is, ru{tI, 1, 2}, CharT{':'}, ru{tM, 1, 2}, + CharT{':'}, rld{S, 1, w}); + checked_set(I, tI, not_a_hour_12_value, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), + not_a_second, is); + ws(is); + auto nm = detail::ampm_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + checked_set(p, static_cast(i), not_a_ampm, is); +#endif + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'R': + if (command) + { + if (modified == CharT{}) + { + int tH = not_a_hour; + int tM = not_a_minute; + read(is, ru{tH, 1, 2}, CharT{'\0'}, CharT{':'}, CharT{'\0'}, + ru{tM, 1, 2}, CharT{'\0'}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'S': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'E'}) +#endif + { + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + long double S; + read(is, rld{S, 1, width == -1 ? w : static_cast(width)}); + checked_set(s, round_i(duration{S}), + not_a_second, is); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + checked_set(s, duration_cast(seconds{tm.tm_sec}), + not_a_second, is); + is.setstate(err); + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'T': + if (command) + { + if (modified == CharT{}) + { + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int tH = not_a_hour; + int tM = not_a_minute; + long double S; + read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, + CharT{':'}, rld{S, 1, w}); + checked_set(H, tH, not_a_hour, is); + checked_set(M, tM, not_a_minute, is); + checked_set(s, round_i(duration{S}), + not_a_second, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'Y': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#else + if (modified != CharT{'O'}) +#endif + { + int tY = not_a_year; + read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}); + checked_set(Y, tY, not_a_year, is); + } +#if !ONLY_C_LOCALE + else if (modified == CharT{'E'}) + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + is.setstate(err); + } +#endif + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'y': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + { + int ty = not_a_2digit_year; + read(is, ru{ty, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(y, ty, not_a_2digit_year, is); + } +#if !ONLY_C_LOCALE + else + { + ios::iostate err = ios::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + checked_set(Y, tm.tm_year + 1900, not_a_year, is); + is.setstate(err); + } +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'g': + if (command) + { + if (modified == CharT{}) + { + int tg = not_a_2digit_year; + read(is, ru{tg, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(g, tg, not_a_2digit_year, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'G': + if (command) + { + if (modified == CharT{}) + { + int tG = not_a_year; + read(is, rs{tG, 1, width == -1 ? 4u : static_cast(width)}); + checked_set(G, tG, not_a_year, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'U': + if (command) + { + if (modified == CharT{}) + { + int tU = not_a_week_num; + read(is, ru{tU, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(U, tU, not_a_week_num, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'V': + if (command) + { + if (modified == CharT{}) + { + int tV = not_a_week_num; + read(is, ru{tV, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(V, tV, not_a_week_num, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'W': + if (command) + { + if (modified == CharT{}) + { + int tW = not_a_week_num; + read(is, ru{tW, 1, width == -1 ? 2u : static_cast(width)}); + checked_set(W, tW, not_a_week_num, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'E': + case 'O': + if (command) + { + if (modified == CharT{}) + { + modified = *fmt; + } + else + { + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + } + else + read(is, *fmt); + break; + case '%': + if (command) + { + if (modified == CharT{}) + read(is, *fmt); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + command = fmt; + break; + case 'z': + if (command) + { + int tH, tM; + minutes toff = not_a_offset; + bool neg = false; + auto ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) + { + auto c = static_cast(Traits::to_char_type(ic)); + if (c == '-') + neg = true; + } + if (modified == CharT{}) + { + read(is, rs{tH, 2, 2}); + if (!is.fail()) + toff = hours{std::abs(tH)}; + if (is.good()) + { + ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) + { + auto c = static_cast(Traits::to_char_type(ic)); + if ('0' <= c && c <= '9') + { + read(is, ru{tM, 2, 2}); + if (!is.fail()) + toff += minutes{tM}; + } + } + } + } + else + { + read(is, rs{tH, 1, 2}); + if (!is.fail()) + toff = hours{std::abs(tH)}; + if (is.good()) + { + ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) + { + auto c = static_cast(Traits::to_char_type(ic)); + if (c == ':') + { + (void)is.get(); + read(is, ru{tM, 2, 2}); + if (!is.fail()) + toff += minutes{tM}; + } + } + } + } + if (neg) + toff = -toff; + checked_set(temp_offset, toff, not_a_offset, is); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'Z': + if (command) + { + if (modified == CharT{}) + { + std::basic_string buf; + while (is.rdstate() == std::ios::goodbit) + { + auto i = is.rdbuf()->sgetc(); + if (Traits::eq_int_type(i, Traits::eof())) + { + is.setstate(ios::eofbit); + break; + } + auto wc = Traits::to_char_type(i); + auto c = static_cast(wc); + // is c a valid time zone name or abbreviation character? + if (!(CharT{1} < wc && wc < CharT{127}) || !(isalnum(c) || + c == '_' || c == '/' || c == '-' || c == '+')) + break; + buf.push_back(c); + is.rdbuf()->sbumpc(); + } + if (buf.empty()) + is.setstate(ios::failbit); + checked_set(temp_abbrev, buf, {}, is); + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + default: + if (command) + { + if (width == -1 && modified == CharT{} && '0' <= *fmt && *fmt <= '9') + { + width = static_cast(*fmt) - '0'; + while ('0' <= fmt[1] && fmt[1] <= '9') + width = 10*width + static_cast(*++fmt) - '0'; + } + else + { + if (modified == CharT{}) + read(is, CharT{'%'}, width, *fmt); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + } + else // !command + { + if (isspace(static_cast(*fmt))) + { + // space matches 0 or more white space characters + if (is.good()) + ws(is); + } + else + read(is, *fmt); + } + break; + } + } + // is.fail() || *fmt == CharT{} + if (is.rdstate() == ios::goodbit && command) + { + if (modified == CharT{}) + read(is, CharT{'%'}, width); + else + read(is, CharT{'%'}, width, modified); + } + if (!is.fail()) + { + if (y != not_a_2digit_year) + { + // Convert y and an optional C to Y + if (!(0 <= y && y <= 99)) + goto broken; + if (C == not_a_century) + { + if (Y == not_a_year) + { + if (y >= 69) + C = 19; + else + C = 20; + } + else + { + C = (Y >= 0 ? Y : Y-100) / 100; + } + } + int tY; + if (C >= 0) + tY = 100*C + y; + else + tY = 100*(C+1) - (y == 0 ? 100 : y); + if (Y != not_a_year && Y != tY) + goto broken; + Y = tY; + } + if (g != not_a_2digit_year) + { + // Convert g and an optional C to G + if (!(0 <= g && g <= 99)) + goto broken; + if (C == not_a_century) + { + if (G == not_a_year) + { + if (g >= 69) + C = 19; + else + C = 20; + } + else + { + C = (G >= 0 ? G : G-100) / 100; + } + } + int tG; + if (C >= 0) + tG = 100*C + g; + else + tG = 100*(C+1) - (g == 0 ? 100 : g); + if (G != not_a_year && G != tG) + goto broken; + G = tG; + } + if (Y < static_cast(year::min()) || Y > static_cast(year::max())) + Y = not_a_year; + bool computed = false; + if (G != not_a_year && V != not_a_week_num && wd != not_a_weekday) + { + year_month_day ymd_trial = sys_days(year{G-1}/December/Thursday[last]) + + (Monday-Thursday) + weeks{V-1} + + (weekday{static_cast(wd)}-Monday); + if (Y == not_a_year) + Y = static_cast(ymd_trial.year()); + else if (year{Y} != ymd_trial.year()) + goto broken; + if (m == not_a_month) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == not_a_day) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + computed = true; + } + if (Y != not_a_year && U != not_a_week_num && wd != not_a_weekday) + { + year_month_day ymd_trial = sys_days(year{Y}/January/Sunday[1]) + + weeks{U-1} + + (weekday{static_cast(wd)} - Sunday); + if (Y == not_a_year) + Y = static_cast(ymd_trial.year()); + else if (year{Y} != ymd_trial.year()) + goto broken; + if (m == not_a_month) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == not_a_day) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + computed = true; + } + if (Y != not_a_year && W != not_a_week_num && wd != not_a_weekday) + { + year_month_day ymd_trial = sys_days(year{Y}/January/Monday[1]) + + weeks{W-1} + + (weekday{static_cast(wd)} - Monday); + if (Y == not_a_year) + Y = static_cast(ymd_trial.year()); + else if (year{Y} != ymd_trial.year()) + goto broken; + if (m == not_a_month) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == not_a_day) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + computed = true; + } + if (j != not_a_doy && Y != not_a_year) + { + auto ymd_trial = year_month_day{local_days(year{Y}/1/1) + days{j-1}}; + if (m == 0) + m = static_cast(static_cast(ymd_trial.month())); + else if (month(static_cast(m)) != ymd_trial.month()) + goto broken; + if (d == 0) + d = static_cast(static_cast(ymd_trial.day())); + else if (day(static_cast(d)) != ymd_trial.day()) + goto broken; + j = not_a_doy; + } + auto ymd = year{Y}/m/d; + if (ymd.ok()) + { + if (wd == not_a_weekday) + wd = static_cast((weekday(sys_days(ymd)) - Sunday).count()); + else if (wd != static_cast((weekday(sys_days(ymd)) - Sunday).count())) + goto broken; + if (!computed) + { + if (G != not_a_year || V != not_a_week_num) + { + sys_days sd = ymd; + auto G_trial = year_month_day{sd + days{3}}.year(); + auto start = sys_days((G_trial - years{1})/December/Thursday[last]) + + (Monday - Thursday); + if (sd < start) + { + --G_trial; + if (V != not_a_week_num) + start = sys_days((G_trial - years{1})/December/Thursday[last]) + + (Monday - Thursday); + } + if (G != not_a_year && G != static_cast(G_trial)) + goto broken; + if (V != not_a_week_num) + { + auto V_trial = duration_cast(sd - start).count() + 1; + if (V != V_trial) + goto broken; + } + } + if (U != not_a_week_num) + { + auto start = sys_days(Sunday[1]/January/ymd.year()); + auto U_trial = floor(sys_days(ymd) - start).count() + 1; + if (U != U_trial) + goto broken; + } + if (W != not_a_week_num) + { + auto start = sys_days(Monday[1]/January/ymd.year()); + auto W_trial = floor(sys_days(ymd) - start).count() + 1; + if (W != W_trial) + goto broken; + } + } + } + fds.ymd = ymd; + if (I != not_a_hour_12_value) + { + if (!(1 <= I && I <= 12)) + goto broken; + if (p != not_a_ampm) + { + // p is in [0, 1] == [AM, PM] + // Store trial H in I + if (I == 12) + --p; + I += p*12; + // Either set H from I or make sure H and I are consistent + if (H == not_a_hour) + H = I; + else if (I != H) + goto broken; + } + else // p == not_a_ampm + { + // if H, make sure H and I could be consistent + if (H != not_a_hour) + { + if (I == 12) + { + if (H != 0 && H != 12) + goto broken; + } + else if (!(I == H || I == H+12)) + { + goto broken; + } + } + } + } + if (H != not_a_hour) + { + fds.has_tod = true; + fds.tod = hh_mm_ss{hours{H}}; + } + if (M != not_a_minute) + { + fds.has_tod = true; + fds.tod.m_ = minutes{M}; + } + if (s != not_a_second) + { + fds.has_tod = true; + fds.tod.s_ = detail::decimal_format_seconds{s}; + } + if (j != not_a_doy) + { + fds.has_tod = true; + fds.tod.h_ += hours{days{j}}; + } + if (wd != not_a_weekday) + fds.wd = weekday{static_cast(wd)}; + if (abbrev != nullptr) + *abbrev = std::move(temp_abbrev); + if (offset != nullptr && temp_offset != not_a_offset) + *offset = temp_offset; + } + return is; + } +broken: + is.setstate(ios::failbit); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, year& y, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.year().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + y = fds.ymd.year(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, month& m, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + m = fds.ymd.month(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, day& d, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.day().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + d = fds.ymd.day(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, weekday& wd, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.wd.ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + wd = fds.wd; + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, year_month& ym, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + ym = fds.ymd.year()/fds.ymd.month(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, month_day& md, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok() || !fds.ymd.day().ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + md = fds.ymd.month()/fds.ymd.day(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + year_month_day& ymd, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = std::chrono::seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.ok()) + is.setstate(std::ios::failbit); + if (!is.fail()) + ymd = fds.ymd; + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + sys_time& tp, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = typename std::common_type::type; + using detail::round_i; + std::chrono::minutes offset_local{}; + auto offptr = offset ? offset : &offset_local; + fields fds{}; + fds.has_tod = true; + from_stream(is, fmt, fds, abbrev, offptr); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(std::ios::failbit); + if (!is.fail()) + tp = round_i(sys_days(fds.ymd) - *offptr + fds.tod.to_duration()); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + local_time& tp, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using CT = typename std::common_type::type; + using detail::round_i; + fields fds{}; + fds.has_tod = true; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(std::ios::failbit); + if (!is.fail()) + tp = round_i(local_seconds{local_days(fds.ymd)} + fds.tod.to_duration()); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + std::chrono::duration& d, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using Duration = std::chrono::duration; + using CT = typename std::common_type::type; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.has_tod) + is.setstate(std::ios::failbit); + if (!is.fail()) + d = std::chrono::duration_cast(fds.tod.to_duration()); + return is; +} + +template , + class Alloc = std::allocator> + struct parse_manip +{ + const std::basic_string format_; + Parsable& tp_; + std::basic_string* abbrev_; + std::chrono::minutes* offset_; + +public: + parse_manip(std::basic_string format, Parsable& tp, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) + : format_(std::move(format)) + , tp_(tp) + , abbrev_(abbrev) + , offset_(offset) + {} + +}; + +template +std::basic_istream& +operator>>(std::basic_istream& is, + const parse_manip& x) +{ + return from_stream(is, x.format_.c_str(), x.tp_, x.abbrev_, x.offset_); +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp) +-> decltype(from_stream(std::declval&>(), + format.c_str(), tp), + parse_manip{format, tp}) +{ + return {format, tp}; +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp, + std::basic_string& abbrev) + -> decltype(from_stream(std::declval&>(), + format.c_str(), tp, &abbrev), + parse_manip{format, tp, &abbrev}) +{ + return {format, tp, &abbrev}; +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp, + std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), + format.c_str(), tp, + std::declval*>(), + &offset), + parse_manip{format, tp, nullptr, &offset}) +{ + return {format, tp, nullptr, &offset}; +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp, + std::basic_string& abbrev, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), + format.c_str(), tp, &abbrev, &offset), + parse_manip{format, tp, &abbrev, &offset}) +{ + return {format, tp, &abbrev, &offset}; +} + +// const CharT* formats + +template +inline +auto +parse(const CharT* format, Parsable& tp) +-> decltype(from_stream(std::declval&>(), format, tp), + parse_manip{format, tp}) +{ + return {format, tp}; +} + +template +inline +auto +parse(const CharT* format, Parsable& tp, std::basic_string& abbrev) +-> decltype(from_stream(std::declval&>(), format, + tp, &abbrev), + parse_manip{format, tp, &abbrev}) +{ + return {format, tp, &abbrev}; +} + +template +inline +auto +parse(const CharT* format, Parsable& tp, std::chrono::minutes& offset) +-> decltype(from_stream(std::declval&>(), format, + tp, std::declval*>(), &offset), + parse_manip{format, tp, nullptr, &offset}) +{ + return {format, tp, nullptr, &offset}; +} + +template +inline +auto +parse(const CharT* format, Parsable& tp, + std::basic_string& abbrev, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), format, + tp, &abbrev, &offset), + parse_manip{format, tp, &abbrev, &offset}) +{ + return {format, tp, &abbrev, &offset}; +} + +// duration streaming + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, + const std::chrono::duration& d) +{ + return os << detail::make_string::from(d.count()) + + detail::get_units(typename Period::type{}); +} + +} // namespace date + +#ifdef _MSC_VER +# pragma warning(pop) +#endif + +#ifdef __GNUC__ +# pragma GCC diagnostic pop +#endif + +#endif // DATE_H diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoUtils.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3688340603385a738561d97b9add6f715d47627e --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoUtils.cpp @@ -0,0 +1,449 @@ +// ============================================================================ +// Copyright 2020-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include "ChronoUtils.h" + +// From https://howardhinnant.github.io/date/date.html +// which is part of C++20, but usable in C++11/14/17. +#define HAS_STRING_VIEW 1 +#define HAS_UNCAUGHT_EXCEPTIONS 1 +//#define ONLY_C_LOCALE 1 +#include "ChronoDate.h" + +#include + +#include +#include +#include +#include + +/*! \file oes/core/utils/ChronoUtils.h + * \brief Utilities using standard header. + * + * Additionally, Howard Hinnant's Date-library, now part of C++20 in , + * is supported via the "ChronoDate.h", which can be either used directly, or + * via the wrapper functions from this header which \em parse and \em format + * std::chrono::system_clock::time_point inputs. This allows to benefit from + * these new C++20 APIs, even though we are still on C++17, and avoids depending + * on Boost Date-Time library, which unlike "ChronoDate.h" is not header-only. + */ + +namespace oes::core { + +//============================================================================ + +/*! + * \class HighResTimer oes/core/utils/ChronoUtils.h + * \brief High-resolution timer for simple timing needs. + * \ingroup oes_core_utils + * + * Minimalistic on purpose, notably not providing std::ostream operator, or + * interop with oes::core::Format; simply call .elapsed() to \em print this timer. + * + * Also header-only and \em inline on purpose. + */ + +/*! + * \brief Restart this timer. + */ +void HighResTimer::restart() { + start_ = clock_t::now(); +} + +/*! + * \brief Elapsed time in floating-point seconds. + * \return Elapsed time, since contrustion or last restart(). + */ +double HighResTimer::elapsedSec() const { + clock_t::time_point end = clock_t::now(); + std::chrono::duration sec = end - start_; + return sec.count(); +} + +/*! + * \brief Elapsed time in seconds as a string, with milliseconds precision. + * \return Elapsed time, since contrustion or last restart(). + */ +std::string HighResTimer::elapsed() const { + return fmt::format("{:.3f}s", elapsedSec()); +} + +//============================================================================ + +/*! + * \brief Parses text consisting of a number followed by a duration unit. + * + * Only simple durations made of a single integer, followed by a single duration + * unit, which must be one of "s" for seconds, "ms" for milliseconds, and "min" + * for minutes. (if no unit is provided, milliseconds is assumed). + * + * Examples of values text: "1min", "30s", "500ms", etc... + * + * \param s the string to parse. + * \return the parsed duration, in milliseconds, on success; -1 otherwise, + * typically when \a s is empty, or uses an unknown duration unit, or + * is badly formed. + */ +std::chrono::milliseconds parseSimpleDuration(std::string s) { + s = boost::algorithm::trim_copy(s); + + size_t consumed = 0; + int val = std::stoi(s, &consumed); + if (!consumed) { + return std::chrono::milliseconds{ -1 }; + } + int factor = 1; // assume ms for backward-compatibility. + const bool no_uom = (consumed == s.size()); + if (!no_uom) { + std::string uom = boost::algorithm::trim_copy(s.substr(consumed)); + if (uom == "s") { + factor = 1000; + } else if (uom == "ms") { + factor = 1; + } else if (uom == "min") { + factor = 60 * 1000; + } else { + return std::chrono::milliseconds{ -1 }; + } + } + return std::chrono::milliseconds(val * factor); +} + +/*! + * \brief Parses text consisting of two durations, separated by a comma. + * + * Splits the text into two parts around the comma, and uses parseSimpleDuration() + * for both sides. For convenience, if a single duration is specified (i.e. no + * comma followed by another duration), returns that duration for both members of + * the returned pair. + * + * Examples of values text: "10s,30s", "500ms", "30s, 1min", etc... + * + * \param s the string to parse. + * \return the parsed durations, in milliseconds, on success; -1 otherwise, + * for either or both durations. + */ +std::pair< + std::chrono::milliseconds, + std::chrono::milliseconds +> parseCsvSimpleDurationPair(const std::string& s) { + if (s.empty()) { + return { std::chrono::milliseconds{ -1 }, std::chrono::milliseconds{ -1 } }; + } + + std::vector parts; + namespace ba = boost::algorithm; + ba::split(parts, s, ba::is_any_of(",;/|"), ba::token_compress_off); + + switch (parts.size()) { + case 1: return { parseSimpleDuration(parts[0]), parseSimpleDuration(parts[0]) }; + case 2: return { parseSimpleDuration(parts[0]), parseSimpleDuration(parts[1]) }; + default: return { std::chrono::milliseconds{ -1 } , std::chrono::milliseconds{ -1 } }; + } +} + +//============================================================================ + +//! \brief ISO-8601 date-time format +const std::string& isoDatetimeFormat() { + static const/*expr*/ std::string fmt__("%Y-%m-%dT%T"); + return fmt__; +} + +//! \brief Format used by std::asctime() minus the trailing newline +const std::string& ascDatetimeFormat() { + static const/*expr*/ std::string fmt__("%a %b %e %H:%M:%S %Y"); + return fmt__; +} + +/*! + * \brief Parses a date-time string, to a time_point. + * \param format the format specifiers. E.g. "%Y-%m-%dT%T" for ISO date-time. + * see https://howardhinnant.github.io/date/date.html#from_stream_formatting + * \param s the date-time to parse + * \param[out] tp the resulting time-point + * \return true on successful parsing; false otherwise + */ +bool parseDateTime( + const std::string& format, + const std::string& s, + std::chrono::system_clock::time_point& tp +) { + std::istringstream ss{ s }; + ss >> date::parse(format, tp); + return !ss.fail(); +} + +/*! + * \brief Parses an Extended ISO date-time string, to a time_point. + * \param s the ISO date-time to parse + * \param[out] tp the resulting time-point + * \return true on successful parsing; false otherwise + * + * \note numeric timezones annotation are accepted, + * but do not change the time_point returned + * \note Uses parseDateTime("%Y-%m-%dT%T", s, tp) + */ +bool parseExtendedIsoDateTime( + const std::string& s, + std::chrono::system_clock::time_point& tp +) { + return parseDateTime(isoDatetimeFormat(), s, tp); +} + +/*! + * \brief Parses an Extended ISO date-time string, to duration. + * \param s the ISO \em extended date-time to parse + * \param[out] usec micro-seconds since Epoch + * \return true on successful parsing; false otherwise + * + * \note numeric timezones annotation are accepted, + * but do not change the duration returned + * \note Uses parseExtendedIsoDateTime(s, tp) + */ +bool usecSinceEpochFromExtendedIsoDateTime( + const std::string& s, + std::chrono::microseconds& usec +) { + std::chrono::system_clock::time_point tp; + if (!parseExtendedIsoDateTime(s, tp)) { + return false; + } + std::chrono::system_clock::duration ts = tp.time_since_epoch(); + usec = std::chrono::duration_cast(ts); + return true; +} + +/*! + * \brief Converts a given time_point to string, according to a given format. + * \param format the format specifiers. E.g. "%Y-%m-%dT%T" for ISO date-time. + * see https://howardhinnant.github.io/date/date.html#to_stream_formatting + * \param tp the time-point to output + * \return string representation of time_point \a tp + */ +std::string formatDateTime( + const std::string& format, + const std::chrono::system_clock::time_point& tp +) { + assert(!format.empty()); + return date::format(format, tp); +} + +/*! + * \brief Converts a given time_point to string, using ISO date-time format. + * + * The number of fractional-second digits depends on the platforms + * (typically 7 on Windows, and 9 on Linux). To control those digits, + * use one of the formatExtendedIsoDateTime() overloads instead. + * + * \param tp the time-point to output + * \return string representation of time_point \a tp + * + * \note Uses formatDateTime("%Y-%m-%dT%T", tp); + */ +std::string formatIsoDateTime( + const std::chrono::system_clock::time_point& tp +) { + return date::format(isoDatetimeFormat(), tp); +} + +/*! + * \brief Converts a given time_point to string, using custom format. + * + * Outputs a date-time with a \b fixed number of fractional-second digits, + * with (whole) seconds, milli-seconds, or micro-seconds resolution, and + * flexible rounding behavior (\em floor, \em round, or \em ceil). + * + * By default the system_clock's time_point has 1/10th of microsecond resolution + * on Windows, and nanosecond resolution on Linux. Which does not mean the CPU + * clock \em ticks that fast, but how much resolution each time_point retains. + * + * \param tp the time-point to output + * \param resolution how many fractional-seconds digits to use (0, 3 or 6) + * and how to round the value with just those digits. + * \return string representation of time_point \a tp + * + * \note Uses formatIsoDateTime(tp); + */ +std::string formatDateTime( + const std::string& format, + const std::chrono::system_clock::time_point& tp, + FracSecRes resolution +) { + auto fmt_it = [format](auto dur) { + return date::format(format, dur); + }; + + switch (resolution) { + + // Whole seconds + case FracSecRes::eWholeSecFloor: + return fmt_it(date::floor(tp)); + case FracSecRes::eWholeSecRound: + return fmt_it(date::round(tp)); + case FracSecRes::eWholeSecCeil: + return fmt_it(date::ceil(tp)); + + // Milli-seconds + case FracSecRes::eMilliSecFloor: + return fmt_it(date::floor(tp)); + case FracSecRes::eMilliSecRound: + return fmt_it(date::round(tp)); + case FracSecRes::eMilliSecCeil: + return fmt_it(date::ceil(tp)); + + // Micro-seconds + case FracSecRes::eMicroSecFloor: + return fmt_it(date::floor(tp)); + case FracSecRes::eMicroSecRound: + return fmt_it(date::round(tp)); + case FracSecRes::eMicroSecCeil: + return fmt_it(date::ceil(tp)); + + } // No default on purpose + + assert(false); + return {}; +} + +/*! + * \brief Converts a given time_point to string, using ISO date-time format. + * + * Outputs a date-time with a \b fixed number of fractional-second digits, + * with (whole) seconds, milli-seconds, or micro-seconds resolution, and + * flexible rounding behavior (\em floor, \em round, or \em ceil). + * + * For a variable number of fractional-second digits, see its overload. + * + * \param tp the time-point to output + * \param resolution how many fractional-seconds digits to use (0, 3 or 6) + * and how to round the value with just those digits. + * \return string representation of time_point \a tp + * + * \note Uses formatIsoDateTime(tp); + */ +std::string formatExtendedIsoDateTime( + const std::chrono::system_clock::time_point& tp, + FracSecRes resolution +) { + return formatDateTime(isoDatetimeFormat(), tp, resolution); +} + +/*! + * \brief Converts a given time_point to string, using ISO date-time format. + * + * Wrapper around formatIsoDateTime() yielding a variable number of + * fractional-second digits, for the shortest text representation given + * \a min_fractional_seconds and \a max_fractional_seconds. + * + * Trims trailing fractional-second digits past \a max_fractional_seconds. + * Trims trailing fractional-second zeros, past \a min_fractional_seconds. + * Trims trailing fractional-second \em period, for \em whole-second times. + * These are text-based post-processing on fractional-second only. + * + * Unlike the overload taking a \a FracSecRes argument, this API supports + * only \em floor()-like \em truncation of fractional-seconds, and a variable + * number of fractional-second digits, depending on the time-point \a tp value, + * while the overload always yields the same number of digits, and is more + * flexible in how the fractional-seconds are rounded up or down. + * + * \param tp the time-point to output + * \param min_fractional_seconds (default 0) How many fractional-second digits + * to retain, even when they are zeros + * \param max_fractional_seconds (default 6) How many fractional-second digits + * to retain, even when more would be required to \em exactly represent + * \a tp's value. I.e. truncation. + * \return string representation of time_point \a tp + * + * \note Uses formatIsoDateTime(tp); + */ +std::string formatExtendedIsoDateTime( + const std::chrono::system_clock::time_point& tp, + int min_fractional_seconds, + int max_fractional_seconds +) { + std::string s = formatIsoDateTime(tp); + + // Fixup fractional seconds + // See https://github.com/HowardHinnant/date/issues/621 + size_t dot_pos = s.find('.'); + if (dot_pos != std::string::npos) { + size_t tail_len = s.size() - dot_pos - 1; // excluding dot + + if (int(tail_len) > max_fractional_seconds) { + // trim excess fractional seconds ("floor()"-like) + s.resize(dot_pos + max_fractional_seconds + 1); + tail_len = size_t(max_fractional_seconds); + } + + while (tail_len && s.back() == '0') { + // trim trailing zeros + s.resize(s.size() - 1); + --tail_len; + } + + if (int(tail_len) < min_fractional_seconds) { + // add trailing zeros + s.insert(s.size(), size_t(min_fractional_seconds) - tail_len, '0'); + } + + if (s.back() == '.') { + // trim when no digits behind + s.resize(s.size() - 1); + } + } + + return s; +} + +//============================================================================ + +/*! + * \brief Offset in seconds between local and UTC (GMT timezone) times. + * \return offset to \b add to local time to get UTC. + * \warning Not thread-safe, since relies on std::gmtime and std::localtime. + */ +std::chrono::seconds tzOffset() { + const std::time_t now = std::time(nullptr); + std::tm gtm = *std::gmtime(&now); + std::tm ltm = *std::localtime(&now); + std::time_t ut = std::mktime(<m); + std::time_t gt = std::mktime(>m); + if (ut == -1 || gt == -1) { + assert(false); + return {}; // conversion failed... + } + return std::chrono::seconds(ut - gt); +} + +/*! + * \brief Cached offset in seconds between local and UTC (GMT timezone) times. + * + * When excluding DST (Daylight Saving Time), the time-zone offset is typically constant. + * Calling this function early in main(), when the program is not yet multi-threaded + * works around the tzOffset()'s non-thread-safety, and records the value for the remaining + * duration of the process. + * + * \return cached result of tzOffset() + */ +std::chrono::seconds cachedTzOffset() { + static const auto tz_offset = tzOffset(); + return tz_offset; +} + +//============================================================================ + +} // namespace oes::core diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoUtils.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..9eb5dba657c0de4db09a1e7d1721209900b7ade7 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ChronoUtils.h @@ -0,0 +1,133 @@ +// ============================================================================ +// Copyright 2020-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_CHRONOUTILS_H +#define OES_CORE_UTILS_CHRONOUTILS_H + +#include + +#include +#include + +namespace oes::core { + +//============================================================================ + +struct OES_CORE_UTILS_EXPORT HighResTimer { + using clock_t = std::chrono::high_resolution_clock; + clock_t::time_point start_ = clock_t::now(); + + void restart(); + double elapsedSec() const; + std::string elapsed() const; +}; + +//============================================================================ + +OES_CORE_UTILS_EXPORT +std::chrono::milliseconds parseSimpleDuration(std::string s); + +OES_CORE_UTILS_EXPORT +std::pair< + std::chrono::milliseconds, + std::chrono::milliseconds +> parseCsvSimpleDurationPair(const std::string& s); + +//============================================================================ + +OES_CORE_UTILS_EXPORT +const std::string& isoDatetimeFormat(); + +OES_CORE_UTILS_EXPORT +const std::string& ascDatetimeFormat(); + +OES_CORE_UTILS_EXPORT +bool parseDateTime( + const std::string& format, + const std::string& s, + std::chrono::system_clock::time_point& tp +); + +OES_CORE_UTILS_EXPORT +bool parseExtendedIsoDateTime( + const std::string& s, + std::chrono::system_clock::time_point& tp +); + +OES_CORE_UTILS_EXPORT +bool usecSinceEpochFromExtendedIsoDateTime( + const std::string& s, + std::chrono::microseconds& usec +); + +OES_CORE_UTILS_EXPORT +std::string formatDateTime( + const std::string& format, + const std::chrono::system_clock::time_point& tp +); + +OES_CORE_UTILS_EXPORT +std::string formatIsoDateTime( + const std::chrono::system_clock::time_point& tp +); + +enum class FracSecRes { // Fractional-second Resolution + eWholeSecFloor, eWholeSecRound, eWholeSecCeil, + eMilliSecFloor, eMilliSecRound, eMilliSecCeil, + eMicroSecFloor, eMicroSecRound, eMicroSecCeil, +}; +OES_CORE_UTILS_EXPORT +std::string formatDateTime( + const std::string& format, + const std::chrono::system_clock::time_point& tp, + FracSecRes resolution +); +OES_CORE_UTILS_EXPORT +std::string formatExtendedIsoDateTime( + const std::chrono::system_clock::time_point& tp, + FracSecRes resolution +); + +OES_CORE_UTILS_EXPORT +std::string formatExtendedIsoDateTime( + const std::chrono::system_clock::time_point& tp, + int min_fractional_seconds = 0, + int max_fractional_seconds = 6 // micro-seconds +); + +//============================================================================ + +OES_CORE_UTILS_EXPORT +std::chrono::seconds tzOffset(); + +OES_CORE_UTILS_EXPORT +std::chrono::seconds cachedTzOffset(); + +inline +std::chrono::system_clock::time_point localToUtc(std::chrono::system_clock::time_point local) { + return local - cachedTzOffset(); +} + +inline +std::chrono::system_clock::time_point utcToLocal(std::chrono::system_clock::time_point utc) { + return utc + cachedTzOffset(); +} + +//============================================================================ + +} // namespace oes::core + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Configurator.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Configurator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6a957b30a4b1e5f4d76fb1baa1bffe2a673b24b0 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Configurator.cpp @@ -0,0 +1,137 @@ +// ============================================================================ +// Copyright 2015-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + + std::string mapEnvironmentVariableNameToOptionName(const std::string& envVariableName, const boost::program_options::options_description& optionsDescription) { + const char* result = ""; + /* + * Check if the environment variable's name is prefixed correctly + */ + if (std::string::npos != envVariableName.find(oes::core::Configurator::s_PGenvironmentPrefix_)) { + /* + * Get the name of the variable without the prefix + */ + static const size_t pg_prefix_length = std::strlen(oes::core::Configurator::s_PGenvironmentPrefix_); + const char* env_variable_name_no_prefix = envVariableName.c_str() + pg_prefix_length; + + /* + * Check if this is the expected option's name + */ + if (NULL != optionsDescription.find_nothrow(env_variable_name_no_prefix, /*approx*/false)) + { + result = env_variable_name_no_prefix; + } + } + + return result; + } + + +} + +/*!\class Configurator oes/core/utils/Configurator.h + * \brief Loads the configuration from the command line and from the environment. + * \details Supports the command line arguments that mix the POSIX and GNU extension. + * The support for the different styles is required as PG SC based application may use different styles + * for configuration of the different parts. + */ + +oes::core::Configurator::Configurator (int argc, char* argv[]) : argc_ (argc), argv_ (argv) { + BOOST_ASSERT (argc>=1); + BOOST_ASSERT (argv); +} + +oes::core::Configurator::Configurator (const Configurator& that) : argc_ (that.argc_), argv_ (that.argv_) { + +} + +const char* oes::core::Configurator::s_PGenvironmentPrefix_ = "PG_ARG_"; + +/*!\brief Loads the configuration. + * \details Loads the configuration according to the expected options. The configuration is loaded from the command + * line arguments. If expected option is not present the command line it is loaded from the environment. + * \param options_description the expected options + * then the configuration is propagated to the environment. + * Is usable for configuration propagation to the stand - alone child processes. + * \return boost::optional object that encapsulates the loaded configuration variables map. If the load fails, + * returns uninitialized boost::optional object. This enables to the client code to recognize the situation when the result + * values map shall not be used. + */ +oes::core::Configurator::LoadResult_t +oes::core::Configurator::loadConfiguration (const boost::program_options::options_description& options_description) const { + BOOST_ASSERT (argc_>=1); + BOOST_ASSERT (argv_); + + boost::program_options::variables_map vm; + LoadResult_t result; + try { + /* + * Parse the command line. + */ + boost::program_options::command_line_parser parser (argc_, argv_); + parser.options (options_description); + parser.allow_unregistered (); + boost::program_options::parsed_options options_command_line = parser.run(); + boost::program_options::store (options_command_line, vm); + + /* + * Parse the environment. The options that are possibly not present in the command line + * will be loaded from the environment. + */ + boost::program_options::parsed_options options_environment = + boost::program_options::parse_environment (options_description, boost::bind (mapEnvironmentVariableNameToOptionName, boost::placeholders::_1, options_description)); + boost::program_options::store (options_environment, vm); + + /* + * If there are possibly observers for the options, notify them. + */ + boost::program_options::notify (vm); + + result = vm; + } catch (const std::exception& program_options_exception) { + /* + * The logger is probably not initialized, therefore the error details are reported to the stderr. + */ + std::cerr << "Load configuration failure: " << program_options_exception.what () << '\n'; + } + + return result; +} + +const char* oes::core::Configurator::appPath () const { + BOOST_ASSERT (argc_>=1); + BOOST_ASSERT (argv_); + + return argv_ [0]; +} diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Configurator.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Configurator.h new file mode 100644 index 0000000000000000000000000000000000000000..bd30c3bc6ccce5dd8f0931f959e1ffbc590ab3c1 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Configurator.h @@ -0,0 +1,83 @@ +// ============================================================================ +// Copyright 2015-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_CONFIGURATOR_H_ +#define OES_CORE_UTILS_CONFIGURATOR_H_ + +#include +#include + +#include +#include +#include +#include +#include + +namespace oes { + namespace core { + class OES_CORE_UTILS_EXPORT Configurator { + public: + Configurator (int argc, char* argv[]); + Configurator (const Configurator& that); + + typedef boost::optional LoadResult_t; + + LoadResult_t loadConfiguration (const boost::program_options::options_description&) const; + + const char* appPath () const; + + static const char* s_PGenvironmentPrefix_; + private: + Configurator& operator= (const Configurator&); + + const int argc_; + char** argv_; + }; + + inline std::string toString (const std::string& str) { + return str; + } + + template + struct ConfigurationEnv { + static void push (const char* key, const boost::program_options::variables_map& map) { + if (map [key].empty ()) return; + + const T& val = map [key].as (); + std::string value_name (Configurator::s_PGenvironmentPrefix_); + value_name += key; + const std::string& value_as_str = toString (val); + System::setEnv (value_name, value_as_str); + } + }; + + template<> + struct ConfigurationEnv { + static void push (const char* key, const boost::program_options::variables_map& map) { + if (map.count (key) >= 1 && true == map [key].as ()) { + std::string value_name (Configurator::s_PGenvironmentPrefix_); + value_name += key; + System::setEnv (value_name, "1"); + } + } + }; + } +} + + + + +#endif /* OES_CORE_UTILS_CONFIGURATOR_H_ */ diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/EnumDecl.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/EnumDecl.h new file mode 100644 index 0000000000000000000000000000000000000000..52e2867dba3b8fe5b910c966be9af8315939452f --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/EnumDecl.h @@ -0,0 +1,395 @@ +// ============================================================================ +// Copyright 2013-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_ENUM_DECL_H +#define OES_CORE_UTILS_ENUM_DECL_H + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace oes { +namespace core { + +/*****************************************************************************/ + +// Inspired from SO's 5093460: http://goo.gl/WBLL6 +// OES_DEFINE_ENUM_IMPL_* macros below are implementation details. +#define OES_DEFINE_ENUM_IMPL_TOSTRING(r, data, elem) \ + case elem: return BOOST_PP_STRINGIZE(elem); +#define OES_DEFINE_ENUM_IMPL_VALUES(r, data, elem) elem, +#define OES_DEFINE_ENUM_IMPL_ISVALID(r, data, elem) case elem: return true; +#define OES_DEFINE_ENUM_IMPL_TOENUM(r, data, elem) \ + else if (enum_string == BOOST_PP_STRINGIZE(elem)) { return elem; } + +/*****************************************************************************/ + +/*! + * \brief Macro declaring an enum type, with implicit conversion to/from string. + * \ingroup oes_core_utils + * + * \details + * This macro uses the preprocessor, to declare an enumeration as usual but with + * a slightly altered syntax which allows to implicitly define conversion + * methods of the enum values into their string representations, and from those + * strings back to the enum. + * + * \param enum_type the enum type name. + * \param invalid_enum the \em invalid enum value, returned for invalid enum + * strings or codes. If no such \em invalid value exists, use any invalid + * integer (like -1), to signal the methods converting strings or codes to + * enum values they must throw an exception on bad values, rather than + * returning the known \em invalid enum value. + * \param enum_values the list of enum values, of the form (val1)(val2)..., + * i.e. a Boost.Preprocessor-compatible Sequence. + * + * Example enum declaration: + * \code + * OES_DEFINE_ENUM(MyDomain, -1, (Time)(Depth)(TimeMigrated)); + * \endcode + * + * The above macro expands into: + * \code + * enum MyDomain { Time, Depth, TimeMigrated }; + * + * struct MyDomainEnumTraits { + * typedef MyDomain EnumType; + * typedef std::string (*EnumMapper)(MyDomain); + * + * static inline bool hasInvalid() {...} + * static inline enum_type invalid() {...} + * static inline std::vector values() {...} + * static inline const char* toString(enum_type enum_value) {...} + * static inline enum_type fromString(const std::string& enum_string) {...} + * static inline bool isValidCode(int v_code) {...} + * }; + * \endcode + * + * The generated EnumTraits struct can be used by persistence code to implicitly + * convert the enum values to string for persistence (on save), and back to + * enum values (on load), thanks to the preprocessor-generated toString() and + * fromString() functions. + * + * The preprocessor-generated values() method allows to get all valid enum values + * and for example automaticall ygenerate an GUI option menu, or a persistence + * \em catalog table for that enum in a RDBMS. + * + * \sa OES_DEFINE_ENUM_TRAITS + * \sa enum_values_of + * \sa enum_string_of + * \sa enum_value_of + * + */ +#define OES_DEFINE_ENUM(enum_type, invalid_enum, enum_values) \ + enum /*class*/ enum_type { \ + BOOST_PP_SEQ_ENUM(enum_values) \ + }; \ + \ + struct enum_type##EnumTraits { \ + typedef enum_type EnumType; \ + typedef std::string (*EnumMapper)(enum_type); \ + \ + static inline const char* name() { \ + return BOOST_PP_STRINGIZE(enum_type); \ + } \ + static inline bool hasInvalid() { \ + return isValidCode(invalid_enum); \ + } \ + static inline enum_type invalid() { \ + if (hasInvalid()) { \ + return static_cast(invalid_enum); \ + } \ + throw std::runtime_error( \ + "Unknown " BOOST_PP_STRINGIZE(enum_type) " enum value" \ + ); \ + } \ + static inline std::vector values() { \ + const enum_type value_array[] = { \ + BOOST_PP_SEQ_FOR_EACH( \ + OES_DEFINE_ENUM_IMPL_VALUES, enum_type, enum_values \ + ) \ + }; \ + const size_t count = sizeof(value_array)/sizeof(*value_array); \ + return std::vector(value_array, value_array + count); \ + } \ + static inline const char* toString(enum_type enum_value) { \ + switch (enum_value) { \ + BOOST_PP_SEQ_FOR_EACH( \ + OES_DEFINE_ENUM_IMPL_TOSTRING, enum_type, enum_values \ + ) \ + default: return ""; \ + } \ + } \ + static inline enum_type fromString(const std::string& enum_string) { \ + if (enum_string.empty()) { return invalid(); } \ + BOOST_PP_SEQ_FOR_EACH( \ + OES_DEFINE_ENUM_IMPL_TOENUM, enum_type, enum_values \ + ) \ + return invalid(); \ + } \ + static inline bool isValidCode(int v_code) { \ + switch (v_code) { \ + BOOST_PP_SEQ_FOR_EACH( \ + OES_DEFINE_ENUM_IMPL_TOSTRING, enum_type, enum_values \ + ) \ + default: return false; \ + } \ + } \ + } // struct enum_type##EnumTraits + +/*****************************************************************************/ + +/*! + * \brief Type trait struct to define the relation between a C++ enum and + * a helper \em traits struct implementing conversions from/to strings. + * + * \tparam TEnum a C++ enum type. + * + * Must have the following : + * - typedef to EnumType + * - typedef to MapperType + * - static method with signature: bool hasInvalid() + * - static method with signature: enum_type invalid() + * - static method with signature: std::vector values(); + * - static method with signature: const char* toString(enum_type enum_value) + * - static method with signature: enum_type fromString(const std::string&) + * - static method with signature: bool isValidCode(int v_code) + * + * \tparam the C++ class to persist. + */ +template +struct EnumTypeTraits { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + + // Error below? You are using code that depends on EnumTypeTraits, but did + // not define your enum using OES_DEFINE_ENUM, or forgot to also use the + // OES_DEFINE_ENUM_TRAITS macro (at global scope, outside any namespace!) + BOOST_STATIC_ASSERT(sizeof(typename TEnum::please_specialize_EnumTypeTrait) == 0); +}; + +/*! + * \brief Allows generic template code to access the OES_DEFINE_ENUM-generated + * \em EnumTraits struct, via partial template specialization. + * + * \warning Must be used at global scope only, outside any namespace. + * + * \code + * OES_DEFINE_ENUM(MyGlobalScopeEnum, -1, (Foo)(Bar)); + * + * namespace acme { + * OES_DEFINE_ENUM(MyNamespaceScopeEnum, -1, (Foo)(Bar)); + * + * struct AcmeBuilder { + * OES_DEFINE_ENUM(MyClassScopeEnum, -1, (Foo)(Bar)); + * }; + * } // namespace acme + * + * OES_DEFINE_ENUM_TRAITS(MyGlobalScopeEnum); + * OES_DEFINE_ENUM_TRAITS(acme::MyNamespaceScopeEnum); + * OES_DEFINE_ENUM_TRAITS(acme::AcmeBuilder::MyClassScopeEnum); + * \endcode + */ +#define OES_DEFINE_ENUM_TRAITS(enum_type) \ + namespace oes { namespace core { \ + template<> struct EnumTypeTraits : \ + public enum_type##EnumTraits \ + { \ + BOOST_STATIC_ASSERT((boost::is_enum::value)); \ + }; \ + }} // namespace oes::core + +/*****************************************************************************/ + +/*! + * \brief Gets the textual name of an enum type TEnum. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * + * \return the enum type name, as declared in the code, as the first parameter + * of the OES_DEFINE_ENUM macro. + * + * \note that this name ignores the C++ namespace the enum may be part of. + * This implies that different enums can map to the same name, if those + * enums are declared in different C++ namespace. + */ +template +std::string enum_name_of() { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + return EnumTypeTraits::name(); +} + +/*! + * \brief Gets all valid enum values for enum type TEnum. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * + * \return the vector of valid TEnum values. The values can be converted into + * strings, using enum_string_of() + */ +template +std::vector enum_values_of() { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + return EnumTypeTraits::values(); +} + +/*! + * \brief Converts an enum value into its string equivalent. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * + * \param value the enum value. + * \return the string matching \a value, exactly as defined in the code. + * An empty string, if \a value is not a valid enum value. + * + * \note \b Silently returns an empty string on invalid enum values. + * + * \sa enum_value_of for the reverse conversion. + */ +template +std::string enum_string_of(TEnum value) { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + return EnumTypeTraits::toString(value); +} + +/*! + * \brief Converts an enum value-string into its enum value equivalent. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * + * \param value_string the string corresponding to an enum value. + * \return the enum value \a value_string. If value_string is invalid (empty, + * or uses the wrong case, or a previously defined enum value was removed + * in the OES_DEFINE_ENUM macro that defined the enum type, this method + * either returns the \em invalid enum value, if TEnum has one, otherwise + * this method throws an std::runtime_error. + * \throw std::runtime_error if \a value_string is invalid and TEnum has no + * \em invalid enum value. + * + * \note \b Silently returns the \em invalid enum on invalid \a value_string, + * if TEnum has one. Throws otherwise. + * + * \sa enum_string_of for the reverse conversion. + */ +template +TEnum enum_value_of(const std::string& value_string) { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + return EnumTypeTraits::fromString(value_string); +} + +/*! + * \brief Converts an enum value-string into its enum value equivalent. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * + * \param value_string the string corresponding to an enum value. + * \param[out] value the resulting enumerated value. + * \return the enum value \a value_string. If value_string is invalid (empty, + * or uses the wrong case, or a previously defined enum value was removed + * in the OES_DEFINE_ENUM macro that defined the enum type, this method + * either returns the \em invalid enum value, if TEnum has one, otherwise + * this method throws an std::runtime_error. + * \throw std::runtime_error if \a value_string is invalid and TEnum has no + * \em invalid enum value. + * + * \note \b Silently returns the \em invalid enum on invalid \a value_string, + * if TEnum has one. Throws otherwise. + * + * \sa enum_string_of for the reverse conversion. + */ +template +void set_enum_value_from(const std::string& value_string, TEnum& value) { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + value = enum_value_of(value_string); +} + +/*! + * \brief Determines whether an enum type has a known \em invalid value. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * + * \return true if has an invalid value; false otherwise. + */ +template +bool enum_has_invalid_value() { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + return EnumTypeTraits::hasInvalid(); +} + +/*! + * \brief Gets the known invalid enum of an enum type, if any. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * + * \return the invalid enum value. + * \throw std::runtime_error if TEnum has no known invalid enum value. + */ +template +TEnum enum_invalid_value() { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + return EnumTypeTraits::invalid(); +} + +/*! + * \brief Convenient function to check whether a value is part of an enumerated. + * + * \tparam TEnum the enum type defined using the OES_DEFINE_ENUM macro, + * and exposed to template code using the OES_DEFINE_ENUM_TRAITS macro. + * \param value the value we want to validate. + * + * \return true if \p value is valid; false otherwise. + * + * \note the \p value is explicitly as a enumerated type to ease client code + * so they can call it without the template parameter. For instance: + * \code + * MyEnum type_=... + * oes::core::enum_is_valid_value(type_); + * \endcode + * can be reduced to: + * \code + * oes::core::enum_is_valid_value(type_); + * \endcode + * To validate an arbitrary integer value, you can cast it: + * \code + * oes::core::enum_is_valid_value(static_cast(-1)); + * \endcode + */ +template +bool enum_is_valid_value(TEnum value) { + BOOST_STATIC_ASSERT((boost::is_enum::value)); + return EnumTypeTraits::isValidCode(value); +} + +/*****************************************************************************/ + +} // namespace core +} // namespace oes + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Format.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Format.cpp new file mode 100644 index 0000000000000000000000000000000000000000..011f4dd4f068c82fa146f7bf3ad1aed3a3fbc55d --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Format.cpp @@ -0,0 +1,1064 @@ +// ============================================================================ +// Copyright 2008-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + + +/*! \file oes/core/utils/Format.h + * \brief Format and Formatter. + */ + + +/* TRANSLATOR oes::core::Format */ + +#include +#include + +#include +#include +#include + +#include +#include + +/*! \defgroup oes_core_utils_i18n Internationalization + * \ingroup oes_core_utils + */ + +// Debugging options +#if 0 +#define DEBUG_PARSE(x) \ + std::cout << "File "__FILE__", " << __LINE__ << ": " << x << std::endl +#else +#define DEBUG_PARSE(x) +#endif + +/***************************************************************************/ + +namespace oes { +namespace core { + +/*! \class Format::Argument oes/core/utils/Format.h + * \brief Base class for Format argument. + * \ingroup oes_core_utils_i18n + */ + +/*! + * \brief Creates an empty Format argument. + */ +Format::Argument::Argument() : + refcount_(0) +{ +} + +/*! + * \brief Destructor. + */ +Format::Argument::~Argument() { +} + +/*! + * \brief Adds a reference to this Argument + */ +void Format::Argument::ref() { + refcount_++; + //std::cout << "ref " << (void*)&refcount_ << " -> " << refcount_ << std::endl; +} + +/*! + * \brief Removes a reference to this FormattedArgument + * + * If the reference count reaches 0, the argument is deleted. + */ +void Format::Argument::unref() { + if( refcount_ > 0 ) { + if( --refcount_ == 0 ) { + //std::cout << "unref " << (void*)&refcount_ << " -> delete" << std::endl; + delete this; + } + } else { + std::cerr << "Internal error: attempt to unref released FormattedArgument: " << this << std::endl; + abort(); + } +} + +/*! \typedef std::vector< Argument* > Format::ArgumentList + * \brief List of arguments passed to the format + * \ingroup oes_core_utils_i18n + */ + +/***************************************************************************/ + +/*! \class Format::SimpleArgument oes/core/utils/Format.h + * \brief Format argument for C++ primitive types type. + * \ingroup oes_core_utils_i18n + */ + +/*! \fn Format::SimpleArgument::SimpleArgument(ValueType const& value) + * Creates a simple argument form \p value. + * \param[in] value value of the argument of type ValueType + */ + +/*! \fn Format::SimpleArgument::~SimpleArgument() + * Destructor. + */ + +/*! \fn Format::SimpleArgument::writeTo(Formatter& formatter) + * Formats the argument using the \p formatter. + */ + +/***************************************************************************/ + +/*! \class Format::FormattedArgument oes/core/utils/Format.h + * \brief Format argument for formatted arguments. + * \ingroup oes_core_utils_i18n + */ + +/*! + * \brief Constructor + */ +Format::FormattedArgument::FormattedArgument(const String& formatString) : + formatString_(formatString) +{ +} + +/*! + * \brief Destructor + * \details Unreferences each argument of the list of arguments. + */ +Format::FormattedArgument::~FormattedArgument() { + for( ArgumentList::iterator i = arguments_.begin(); i != arguments_.end(); ++i ) { + (*i)->unref(); + } +} + +/*! + * \brief Adds a argument to the format + */ +void Format::FormattedArgument::addArgument(Argument* argument) { + argument->ref(); + arguments_.push_back(argument); +} + +/*! + * \brief Adds a Format argument to the format + */ +void Format::FormattedArgument::addArgument(const Format& format) { + format.impl_->ref(); + arguments_.push_back(format.impl_); +} + +/*! + * \brief Prints the Format to the given \p formatter. + */ +void Format::FormattedArgument::writeTo(Formatter& formatter) { + formatter.format(formatString_, arguments_); +} + +/*! + * \brief Checks if a format is empty + * \return true if the format string + * and the argument list are empty. + */ +bool Format::FormattedArgument::isEmpty() const { + return formatString_.empty() && arguments_.size() == 0; +} + +/****************************************************************************/ + +/*! \class Format oes/core/utils/Format.h + * \brief Printf like formatter. + * \ingroup oes_core_utils oes_core_utils_i18n + * + * \section format_intro 1. Introduction + * + * C++ streams are powerful tools to compose complex messages that incorporate + * both primitive and custom types by the means of overloading operator <<. + * + * However the nature of C++ streams imposes that non literal string + * arguments be passed separately to the streams, thus breaking the literal + * message into several pieces that are not well suited for + * internationalization (see \ref oes_core_utils_i18n). + * + * \section format_usage 2. Usage + * + * Format uses the same principles as \c sprintf: + * + * - it produces a string by the means of a format string and optional + * arguments + * - optional arguments are formatted according to parameter specifiers in the + * format string + * + * So, with class Format, C++ stream-based messages like this one: + * + * \code + * cout << "I got an error on " << param1 << ". It should be: " + * << good_value << endl; + * \endcode + * + * can be rewritten in the following form: + * + * \code + * Format f("I got an error on %1. It should be: %2\n"); + * f % param % good_value; + * cout << f; + * \endcode + * + * or more concisely: + * + * \code + * Format f("I got an error on %1. It should be: %2\n"); + * cout << f % param % good_value; + * \endcode + * + * or even more concisely: + * + * \code + * cout << Format("I got an error on %1. It should be: %2\n") % param % good_value; + * \endcode + * + * + * \section format_params 3. Parameters and Arguments + * + * \subsection format_params_1 3.1 Parameters + * + * Format parameters are called "positional parameters", because they refer to + * format arguments by their position in the argument list. + * + * The most simple form of positional parameter is \c \%n, where n is the + * position (starting at 1) of the argument in the argument list. + * + * In the example above: + * + * - \%1 denotes the first format argument: \c param + * - \%2 denotes the second format argument: \c good_value + * + * As format parameters start with character \c \%, use the special sequence + * \c \%\% to put a literal character '\%' in the format string as illustrated + * below: + * + * \code + * cout << Format("I'm almost %1%% done\n") % percentage; + * \endcode + * + * \subsection format_params_2 3.2 Arguments + * + * Arguments are passed to the Format by the means of the C++ operator\%(). + * This mimics the convention used in the Boost class \c format, so that boost + * aficionados won't be lost (for more details see + * http://www.boost.org/libs/format/index.html) + * + * Format natively accepts arguments of any C++ primitive types. To add support + * for a custom type, see \ref format_custom. + * + * One of the big advantages of the positional parameters, is that their order + * can be changed in the format string without changing the order of the + * corresponding arguments. This is especially useful for translating messages + * to languages having a different grammatical structure. + * + * \subsection format_params_3 3.3 Multiple Occurrences of Positional Parameters + * + * A same positional parameter can appear several time in the format string. + * This simply format the corresponding argument multiple times in the + * resulting output. + * + * \code + * cout << Format("Value %1 is null. Cannot divide %2 by %1\n") % x % y; + * \endcode + * + * Here, the format has 3 parameters, but 2 arguments only: + * + * - \%1 which denotes argument \c x, is used twice. + * - \%2 which denotes argument \c y, is used only once. + * + * \subsection format_params_4 3.4 Passing a Format as Argument + * + * Format accepts arguments of type Format. This results as if the Format + * argument(s) had been inlined in the containing format string. + * + * The following code: + * + * \code + * Format f1("(%1,%2,%3)") % x1 % y1 % z1; + * Format f2("(%1,%2,%3)") % x2 % y2 % z2; + * cout << Format("Distance between %1 and %2 is %3\n") % f1 % f2 % distance; + * \endcode + * + * is equivalent (but much more convenient) to writing: + * + * \code + * cout << Format("Distance between (%1,%2,%3) and (%4,%5,%6) is %7\n") + * % x1 % y1 % z1 % x2 % y2 % z2 % distance; + * \endcode + * + * + * \subsection format_params_5 3.5 Unformatted arguments + * + * Arguments supplied to the format that are not referenced by a positional + * parameter are called "unformatted arguments" and are simply appended to the + * format output, after all referenced arguments have been formatted. + * + * \code + * cout << Format("Only arg#1 (%1) and arg#3 (%3) are referenced\n") % a % b % c %d; + * \endcode + * + * Here, the second and fourth argument are not referenced in the format + * string. This is equivalent to the following code: + * + * \code + * cout << Format("Only arg#1 (%1) and arg#3 (%3) are referenced\n%2%4") % a % b % c % d; + * \endcode + * + * + * \section format_directives 4. Formatting Directives + * + * \subsection format_directives_1 4.1 Basics + * + * In their most simple form, parameters use the same formatting defaults as + * the C++ streams for the C++ primitive types. + * + * To affect how arguments are converted, parameters can specify additional + * formatting directives. Additional formatting directives are specified + * between vertical bars after the parameter: \c \%n|directives|. + * + * The format of the directives is the same as the one used in \c the function + * printf. For a complete description of the \c printf formatting directives, + * see the \c printf documentation at + * http://www.cplusplus.com/ref/cstdio/printf.html . + * + * Example: the following C++ stream based code + * + * \code + * const char* object_name; + * float value; + * cout << "I got an error in object " << left << setw(16) << object_name + * << ": value should be: " << right << setw(8) << setprecision(2) << value + * << endl; + * \endcode + * + * can be rewritten as follows with the Format class: + * + * \code + * const char* object_name; + * float value; + * cout << Format("I got an error in object %1|-16s|: value should be: %2|8.2f|\n") + * % object_name % value; + * \endcode + * + * \subsection format_directives_2 4.2 Compatibility with printf + * + * The format of the \c printf formatting directives is the following: + * + * \c [flags][width][.precision][modifiers]type + * + * The Format class fully recognizes \c printf formatting directives, except + * one: it does not accept the variable width modifier \c '*'. + * + * Unlike function \c printf, the Format class is a type safe system which + * knows about the type of its arguments. So the type specifiers which are + * mandatory in \c printf can be omitted with Format under certain + * circumstances. + * + * The table below lists the type specifiers which can be omitted in the + * parameter formatting directives. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Basic C++ TypeOptional Type Specifier
boold
charc
wchar_tlc
shorthd
unsigned shorthu
inthd
unsigned inthu
longld
unsigned longlu
signed long longlld
unsigned signed long longllu
floatf
doublelf
long doubleLf
const char*s
const wchar_t*ls
const void*p
+ * + * So the code sample above is strictly equivalent to the following code: + * + * \code + * const char* object_name; + * float value; + * cout << Format("I got an error in object %1|-16|: value should be: %2|8.2|\n") + * % object_name % value; + * \endcode + * + * Note how the type specifiers have been removed. + * + * \subsection format_directives_3 4.3 Compatibility with streams + * + * What C++ streams can do, Format can also do, except for one thing which not + * supported by printf: padding characters. There is currently no + * equivalent to the C++ stream \c setfill manipulator. The only padding + * character supported by printf (and Format) is '0' and only applies to + * integral values. + * + * The table below illustrates the correspondence between standard C++ stream + * manipulators and the corresponding parameter formatting directives in + * Format. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * supported + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Stream ManipulatorFormat DirectiveDescription
setw(n)|n|Print the next value written in a n-characters wide field. + * By default the value is right justified
left|-|Left justify value in its field
setfill(c)|0|Not supported. Except for the modifier flag \c 0 + * which pads '0' characters for integral values.
setprecision(n)|.n| + * Specifies the precision of the value: + * - number of significant digits after the point for floating values + * - the maximum number of characters for string values. + *
fixed|f|Use fixed point notation for floating point values
scientific|e|Use scientific notation for floating point values
no fixed, no scientific|g|Use standard notation for floating point values
dec|d|, |i|, |u|Print integral values in decimal
oct|o|Print integral values in octal
hex|x|Print integral values in hexadecimal
uppercase|X|, |F|, |E|, |G|Print floating point values and hexadecimal digits in uppercase
showbase|#|Print the base prefix for integral values (0x for hexadecimal, 0 for octal)
showpoint|#|Print trailing decimal point and zeros for floating point values
showpos|+|Print the + sign for positive integral values
+ * + * \warning Unlike most C++ stream manipulators, Format directives do not + * affect the Format permanently: they only apply to the positional parameter + * where they are defined. + * + * \section format_custom 5. Adding Support for Custom Types + * + * To add support for a new custom type, you must overload function + * formatArg() for this type as illustrated below: + * + * \code + * class CustomType { + * public: + * CustomType(); + * Type1 param1() const; + * Type2 param2() const; + * }; + * + * Format& formatArg(Format& format, const CustomType& value) { + * return format % (Format("{%1|.2|, %2|-6|}") % value.param1() % value.param2()); + * } + * \endcode + * + * \section format_i18n 6. Internationalization + * + * The default Format constructor does not allow format string to be + * translated: they will not be registered in the translation dictionary by + * the internationalization tools. + * To create a Format from a translatable string, use the static function + * Format::tr() as illustrated below: + * + * \code + * const char* object_name; + * float value; + * cout << Format::tr("I got an error in object %|-16|: value should be: %2|8.2|\n") + * % object_name % value; + * \endcode + * + * This way the format string will be added to the translation dictionary and + * can be translated to the proper language. + * + * + * \section format_output 7. Retrieving Format Output + * + * The result of a Format can be retrieved in a variety of forms: + * + * 1. It can be sent to a C++ output stream + * + * \code + * std::cout << Format::tr("Hello %1\n") % "World"; + * \endcode + * + * 2. You can use convenience copy constructors to construct strings + * from a Format: + * + * \code + * Format f = Format::tr("Hello %1\n") % "World"; + * + * // Use String constructor + * String string(f); + * + * \endcode + * + * 3. You can store the result to a String by the means of + * the function Format::operator>>(): + * + * \code + * Format f = Format::tr("Hello %1\n") % "World"; + * + * String cstring; + * f >> cstring; + * + * \endcode + * + */ + +/*! + * \brief Creates a Format with an empty string. + */ +Format::Format() : + impl_(new FormattedArgument("")) +{ + impl_->ref(); +} + +/*! + * \brief Creates a new Format initialized with \p format string + * \details The format is initialized with the format string \p format + */ +Format::Format( + const String& formatString +) : impl_(new FormattedArgument(formatString)) +{ + impl_->ref(); +} + +/*! + * \brief Copy constructor + * \warning This does a shallow copy of the specified \a format, + * so the copy shares the format string and the argument list with \a format. + */ +Format::Format(const Format& format) : + impl_(format.impl_) +{ + impl_->ref(); +} + +/*! + * \brief Assignment operator + * \warning This does a shallow copy of the specified \a format, + * so the copy shares the format string and the argument list with \a format. + */ +Format& Format::operator=(const Format& format) { + impl_ = format.impl_; + impl_->ref(); + return *this; +} + +/*! \brief Checks if a format is empty. + * \return true if the format string + * and the argument list are empty. + */ +bool Format::isEmpty() const { + return impl_->isEmpty(); +} + +/*! + * \brief Creates a translatable Format + * \details The format is initialized with the format string \p format + * Use this form when you want to create a format with a string that is + * candidate to internationalization + */ +Format Format::tr(const String& formatString) { + return Format(formatString); +} + +/*! + * \brief Destructor + */ +Format::~Format() { + impl_->unref(); +} + +/*! + * \brief Prints the format to the specified Formatter. + */ +void Format::writeTo(Formatter& formatter) const { + impl_->writeTo(formatter); +} + +/*! + * \brief Prints format \p format to the stream \p os. + */ +std::ostream& operator<<(std::ostream& os, const Format& format) { + String string; + format >> string; + return os << string; +} + +/*! \fn Format& Format::operator%(ValueType const& value) + * \brief Format argument inserter + * + * This operator adds argument \p value of type \c ValueType to the current + * Format. \c ValueType must be either a C++ primitive type, or a user type for + * which specific function formatArg() has been defined. + * + * \sa \ref format_custom + */ + +/*! \fn void Format::operator>>(StringType& string) const + * \brief Converts the Format to the specified \p string + * + * Converts the Format to the specified \p string of type \c StringType using + * the appropriate Formatter class. + * + * \internal + * operator>>() consults class FormatTraits to determine which Formatter to use + * for the specified StringType. The formatter is then used to convert this + * format to the desired StringType and store the result to \p string. + */ + +/***************************************************************************/ + +/*! \class FormatTraits oes/core/utils/Format.h + * \internal + * \ingroup oes_core_utils_i18n + * + * This helper class defines a mapping between a string type and the Formatter + * type which is able to convert a Format to the desired string type. This + * class is used by the template member function Format::operator>>(). + */ + +/***************************************************************************/ + +/*! \class Formatter oes/core/utils/Format.h + * \brief Formats a formatted string. + * \ingroup oes_core_utils_i18n + * + * This is a base class for formatter. + */ + +/*! + * \brief Constructor. + */ +Formatter::Formatter() { +} + +/*! + * \brief Destructor. + */ +Formatter::~Formatter() { +} + +/*! + * \brief Adds a \p prefix to the Format. + * \param[in] prefix string prepended to the format result + */ +void Formatter::formatPrefix(const String& prefix) { + (*this) << prefix; +} + +/***************************************************************************/ + +/*! \class StdFormatter oes/core/utils/Format.h + * \ingroup oes_core_utils_i18n + * \brief Formats a formatted string. + * + * The string is formatted in a ostrstream. + */ + +/*! + * \brief Constructor + */ +StdFormatter::StdFormatter() : + Formatter() { +} + +/*! + * \brief Destructor + */ +StdFormatter::~StdFormatter() { +} + +/*! + * \brief Returns the formatted string + */ +String StdFormatter::string() { + return stream_.str(); +} + +Formatter& StdFormatter::operator << (bool b) { + stream_ << b; + return *this; +} + +Formatter& StdFormatter::operator << (signed char c) { + stream_ << c; + return *this; +} + +Formatter& StdFormatter::operator << (unsigned char c) { + stream_ << c; + return *this; +} + +Formatter& StdFormatter::operator << (signed short s) { + stream_ << s; + return *this; +} + +Formatter& StdFormatter::operator << (unsigned short s) { + stream_ << s; + return *this; +} + +Formatter& StdFormatter::operator << (signed int i) { + stream_ << i; + return *this; +} + +Formatter& StdFormatter::operator << (unsigned int i) { + stream_ << i; + return *this; +} + +Formatter& StdFormatter::operator << (signed long l) { + stream_ << l; + return *this; +} + +Formatter& StdFormatter::operator << (unsigned long l) { + stream_ << l; + return *this; +} + +Formatter& StdFormatter::operator << (signed long long i) { + stream_ << i; + return *this; +} + +Formatter& StdFormatter::operator << (unsigned long long i) { + stream_ << i; + return *this; +} + +Formatter& StdFormatter::operator << (float f) { + stream_ << f; + return *this; +} + +Formatter& StdFormatter::operator << (double d) { + stream_ << d; + return *this; +} + +Formatter& StdFormatter::operator << (const char* c) { + stream_ << c; + return *this; +} + +Formatter& StdFormatter::operator << (const String& s) { + stream_ << s; + return *this; +} + +Formatter& StdFormatter::operator << (const String* s) { + stream_ << s; + return *this; +} + +Formatter& StdFormatter::operator << (const void* ptr) { + stream_ << ptr; + return *this; +} + +Formatter& StdFormatter::write(const void* ptr, size_t size) { + stream_.write(reinterpret_cast(ptr), size); + return *this; +} + +/*! + * \brief Parses the formatted string. + */ +void StdFormatter::format( + const String& format, + const Format::ArgumentList& arguments +) { + if( format.empty() ) { + return; + } + + // Build these regexps only once + // "Fix" for PR 27904. + // The immortal hack below only works reasonably because they are immediately + // assigned to static references. Never use this code under any other + // circumstances. + + static const boost::regex re1("%(%|\\d+)"); + static const boost::regex re2("^\\|(#?0?-? ?'?\\+?)(\\d*)(\\.\\d+)?(hh|ll|[hlLqjzt])?([diouxXeEfFgGcsp]?)\\|"); + + DEBUG_PARSE( "Reinitializing format with: " << format ); + + // argumentUsed keeps the indices of used arguments + // NOTE: add + 1 to the number of argument to make sure that we allocate + // at least one element in array argumentUsed + + std::vector argumentUsed(arguments.size()+1, false); + + String::const_iterator start = format.begin(); + String::const_iterator end = format.end(); + boost::smatch result; + + while( regex_search(start, end, result, re1) ) { + + start = result.suffix().first; + + // Output literal prefix + + stream_ << result.prefix().str(); + + DEBUG_PARSE( "Detected parameter: " << result.str() ); + DEBUG_PARSE( "Pending literal = '" << result.prefix().str() << "'" ); + + if( result.str()[1] == '%' ) { + DEBUG_PARSE( "Detected double '%' - keep '%'" ); + stream_ << "%"; + continue; + } + + // Get the parameter index + + size_t paramIndex = std::atol(result.str(1).c_str()); + if( paramIndex > arguments.size() ) { + OES_ERROR(Format("Parameter %%%1 has no value in format string: %2\n") % paramIndex % format); + stream_ << "%" << paramIndex; + continue; + } + + // Mark the parameter argument as used + + argumentUsed[paramIndex - 1] = true; + + // Parse option parameter formatting directives + + if( ! regex_search(start, end, result, re2)) { + // Format the argument using the current stream configuration + arguments.at(paramIndex-1)->writeTo(*this); + continue; + } + + start = result.suffix().first; + + DEBUG_PARSE( "Got formatting directives: " << result.str() ); + DEBUG_PARSE( "- flags=" << result.str(1) ); + DEBUG_PARSE( "- width=" << result.str(2) ); + DEBUG_PARSE( "- precision=" << result.str(3) ); + DEBUG_PARSE( "- conversion=" << result.str(5) ); + + // Save the state of the stream before configuration + + struct State { + std::streamsize width; + std::streamsize precision; + int flags; + char fill; + } state; + + state.width = stream_.width(); + state.precision = stream_.precision(); + state.flags = stream_.flags(); + state.fill = stream_.fill(); + + // Parse the flag characters + + String flags = result.str(1); + for( String::iterator p = flags.begin(); p != flags.end(); ++p ) { + switch(*p) { + case '#' : + DEBUG_PARSE( "set showbase" ); + DEBUG_PARSE( "set showpoint" ); + DEBUG_PARSE( "set internal" ); + stream_.setf(std::ios::showbase); + stream_.setf(std::ios::showpoint); + stream_.setf(std::ios::internal, std::ios::adjustfield); + break; + case '0' : + DEBUG_PARSE( "set fill='0'" ); + stream_.fill('0'); + break; + case '-' : + DEBUG_PARSE( "set left" ); + stream_.setf(std::ios::left, std::ios::adjustfield); + break; + case '+' : + DEBUG_PARSE( "set showpos" ); + stream_.setf(std::ios::showpos); + break; + } + } + + // Parse the field width + + if (result[2].matched) try { + int width = atoi(result.str(2).c_str()); + stream_.width(width); + DEBUG_PARSE( "set width=" << width ); + } catch(...) { + //std::cout << "** Format error in " << format << ": could not decode width field: " << result.str(2) << std::endl; + } + + // Parse the field precision + + if (result[3].matched) try { + boost::smatch::const_reference m = result[3]; + int precision = atoi(String(m.first+1, m.second).c_str()); + stream_.precision(precision); + DEBUG_PARSE( "set precision=" << precision ); + } catch(...) { + //std::cout << "** Format error in " << format << ": could not decode precision field: " << result.str(3) << std::endl; + } + + // Parse the length modifier + // present in submatch result[4] but not needed here + + // Parse the conversion specifier + + String conversion = result.str(5); + for( String::iterator p = conversion.begin(); p != conversion.end(); ++p ) { + switch(*p) { + case 'd' : + case 'i' : + case 'u' : + DEBUG_PARSE( "set dec" ); + stream_.setf(std::ios::dec, std::ios::basefield); + break; + case 'X' : + DEBUG_PARSE( "set uppercase" ); + stream_.setf(std::ios::uppercase); + // FALL THROUGH + case 'x' : + DEBUG_PARSE( "set hex" ); + stream_.setf(std::ios::hex, std::ios::basefield); + break; + case 'o' : + DEBUG_PARSE( "set oct" ); + stream_.setf(std::ios::oct, std::ios::basefield); + break; + case 'F' : + DEBUG_PARSE( "set uppercase" ); + stream_.setf(std::ios::uppercase); + // FALL THROUGH + case 'f' : + DEBUG_PARSE( "set fixed" ); + stream_.setf(std::ios::fixed, std::ios::floatfield); + break; + case 'E' : + DEBUG_PARSE( "set uppercase" ); + stream_.setf(std::ios::uppercase); + // FALL THROUGH + case 'e' : + DEBUG_PARSE( "set scientific" ); + stream_.setf(std::ios::scientific, std::ios::floatfield); + break; + case 'G' : + DEBUG_PARSE( "set uppercase" ); + stream_.setf(std::ios::uppercase); + // FALL THROUGH + case 'g' : + // correspond to standard output: no fixed, no scientific + break; + } + } + + // Format the argument + + arguments.at(paramIndex-1)->writeTo(*this); + + // Restores the stream according to the old state. + + stream_.width(state.width); + stream_.precision(state.precision); + stream_.flags(static_cast(state.flags)); + stream_.fill(state.fill); + } + + // Output pending literal + + stream_ << String(start, end); + + // Append unused arguments here + + for( size_t i = 0; i < arguments.size(); ++i ) { + // Check if this argument has been formatted + if( ! argumentUsed[i] ) { + arguments.at(i)->writeTo(*this); + } + } +} + +/***************************************************************************/ + +} // namespace core +} // namespace oes diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Format.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Format.h new file mode 100644 index 0000000000000000000000000000000000000000..a02692ce31d2d61c4bf8625a803d16ef368d589e --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Format.h @@ -0,0 +1,326 @@ +// ============================================================================ +// Copyright 2008-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_FORMAT_H +#define OES_CORE_UTILS_FORMAT_H + +#include +#include +#include + +#include +#include + + +/*****************************************************************************/ + +namespace oes { +namespace core { + +class OES_CORE_UTILS_EXPORT Format { +public: + Format(); + Format(const String& formatString); + Format(const Format& format); + Format& operator=(const Format& format); + ~Format(); + + bool isEmpty() const; + + static Format tr(const String& formatString); + + template + Format& operator % (ValueType const& value); + + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const Format& m); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, bool value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, signed char value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned char value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, short value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned short value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, int value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned int value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, long value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned long value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, signed long long value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned long long value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, float value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, double value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const void* value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const char* value); + friend OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const String& value); + + template + void operator>>(StringType& string) const; + + class OES_CORE_UTILS_EXPORT Argument { + public: + virtual void writeTo(Formatter& formatter) = 0; + void ref(); + void unref(); + protected: + Argument(); + virtual ~Argument(); + private: + size_t refcount_; + }; + + typedef std::vector ArgumentList; + +private: + void writeTo(Formatter& formatter) const; + + template + class SimpleArgument : public Argument { + public: + SimpleArgument(ValueType const& value); + virtual void writeTo(Formatter& formatter); + protected: + virtual ~SimpleArgument(); + private: + ValueType value_; + }; + + class OES_CORE_UTILS_EXPORT FormattedArgument : public Argument { + public: + FormattedArgument(const String& formatString); + virtual void writeTo(Formatter& formatter); + + bool isEmpty() const; + void addArgument(Argument* argument); + void addArgument(const Format& format); + + protected: + virtual ~FormattedArgument(); + private: + String formatString_; + ArgumentList arguments_; + }; + + FormattedArgument* impl_; +}; + +OES_CORE_UTILS_EXPORT std::ostream& operator << (std::ostream& os, const Format& format); + +/*****************************************************************************/ + +class OES_CORE_UTILS_EXPORT Formatter { +public: + virtual ~Formatter(); + + virtual Formatter& operator << (bool) = 0; + virtual Formatter& operator << (signed char c) = 0; + virtual Formatter& operator << (unsigned char c) = 0; + virtual Formatter& operator << (signed short i) = 0; + virtual Formatter& operator << (unsigned short i) = 0; + virtual Formatter& operator << (signed int i) = 0; + virtual Formatter& operator << (unsigned int i) = 0; + virtual Formatter& operator << (signed long i) = 0; + virtual Formatter& operator << (unsigned long i) = 0; + virtual Formatter& operator << (signed long long i) = 0; + virtual Formatter& operator << (unsigned long long i) = 0; + virtual Formatter& operator << (float f) = 0; + virtual Formatter& operator << (double f) = 0; + virtual Formatter& operator << (const char* s) = 0; + virtual Formatter& operator << (const String& s) = 0; + virtual Formatter& operator << (const String* s) = 0; + virtual Formatter& operator << (const void* ptr) = 0; + virtual Formatter& write(const void* ptr, size_t size) = 0; + + virtual void format(const String& format, const Format::ArgumentList& arguments) = 0; + virtual void formatPrefix(const String& prefix); + +protected: + Formatter(); +}; + +/***************************************************************************/ + +class OES_CORE_UTILS_EXPORT StdFormatter : public Formatter { +public: + StdFormatter(); + virtual ~StdFormatter(); + + String string(); + + virtual Formatter& operator << (bool); + virtual Formatter& operator << (signed char c); + virtual Formatter& operator << (unsigned char c); + virtual Formatter& operator << (signed short i); + virtual Formatter& operator << (unsigned short i); + virtual Formatter& operator << (signed int i); + virtual Formatter& operator << (unsigned int i); + virtual Formatter& operator << (signed long i); + virtual Formatter& operator << (unsigned long i); + virtual Formatter& operator << (signed long long i); + virtual Formatter& operator << (unsigned long long i); + virtual Formatter& operator << (float f); + virtual Formatter& operator << (double f); + virtual Formatter& operator << (const char* s); + virtual Formatter& operator << (const String& s); + virtual Formatter& operator << (const String* s); + virtual Formatter& operator << (const void* ptr); + virtual Formatter& write(const void* ptr, size_t size); + +private: + virtual void format(const String& format, const Format::ArgumentList& arguments); + std::ostringstream stream_; +}; + +/***************************************************************************/ + +template +struct FormatTraits { +}; + +template<> +struct FormatTraits { + typedef StdFormatter FormatterType; +}; + +/*****************************************************************************/ + +template +inline Format::SimpleArgument::SimpleArgument(ValueType const& value) : + value_(value) { +} + +template +Format::SimpleArgument::~SimpleArgument() { +} + +template +void Format::SimpleArgument::writeTo(Formatter& formatter) { + formatter << value_; +} + +template +void Format::operator>>(StringType& string) const { + typedef typename FormatTraits::FormatterType FormatterType; + FormatterType formatter; + this->writeTo(formatter); + string = formatter.string(); +} + +template +inline Format& Format::operator % (ValueType const& value) { + +#if defined(_WIN32) + #pragma warning( push ) + #pragma warning(disable: 4267) // conversion from 'size_t' to 'type', possible loss of data +#endif + + return formatArg(*this, value); + +#if defined(_WIN32) + #pragma warning( pop ) +#endif + +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const Format& m) { + format.impl_->addArgument(m); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, bool value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, signed char value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned char value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, short value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned short value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, int value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned int value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, long value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned long value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, signed long long value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, unsigned long long value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, float value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, double value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const void* value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const char* value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const String& value) { + format.impl_->addArgument(new Format::SimpleArgument(value)); + return format; +} + +inline OES_CORE_UTILS_EXPORT Format& formatArg(Format& format, const String* value) { + return format % (value == nullptr ? "(nullptr)" : *value); +} + +} // namespace core +} // namespace oes + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5e32048861c9192795168baf320f6b0c6f62659d --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.cpp @@ -0,0 +1,832 @@ +// ============================================================================ +// Copyright 2013-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#include + +//#define OES_CORE_UTILS_AVOID_BOOST 1 + +#include + +#include +#include +#include +#include +#ifndef OES_CORE_UTILS_AVOID_BOOST +#include // use "native" thread_local instead +#include +#else +#include +#include +#endif +#include + +/*! \file oes/core/utils/Guid.h + * \brief Defines Guid class and associated free functions. + */ + +/* TRANSLATOR oes::core::Guid */ + +namespace { + +static const unsigned char zeros__[16] = {0}; //< array of 16 zero bytes. + +/*! + * \brief Converts a \em nibble into its (lowercase) hexadecimal encoding. + * hexadecimal letter encodes. + * \param nibble half a byte, i.e. 4 bits, encoding a single hexadecimal letter. + * \return the hexadecimal letter ([0-9a-f]) for \a nibble in range [0, 15]. + * \precondition 0 <= nibble < 16 + */ +static char to_char(unsigned char nibble) { + static const char hex[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + return hex[nibble]; +} + +/*! + * \brief Converts an hexadecimal letter to its decimal equivalent. + * + * \param c the hexadecimal letter + * \return the \em nibble decimal in range [0, 15] + * + * \precondition isxdigit(c) == true + */ +static inline int to_nible(int c) { + return + (c <= '9')? c - '0': + (c <= 'F')? c - 'A' + 10: + c - 'a' + 10; +} + +/*! + * \brief Converts a pair of hexadecimal letters into a [0, 255] byte. + * + * \param high the hexadecimal letter encoding the upper 4 bits. + * \param low the hexadecimal letter encoding the lower 4 bits. + * \return the byte in range [0, 255]. + * + * \note this method is equivalent to using sscanf("%02x", &byte); + * \precondition (isxdigit(low) && isxdigit(high)) == true + */ +static inline unsigned char to_ubyte(int high, int low) { + return static_cast( + (to_nible(high) << 4) + // the high 4-bits of the byte + to_nible(low) // the low 4-bits of the byte + ); +} + +/*! + * \brief Converts a group of 4 hexadecimal letters into 2 bytes. + * + * \param c1 the 1st hexadecimal letter + * \param c2 the 2nd hexadecimal letter + * \param c2 the 3rd hexadecimal letter + * \param c2 the 4th hexadecimal letter + * \param[out] b1 the byte encoded by \a c1 and \a c2 + * \param[out] b2 the byte encoded by \a c3 and \a c4 + * \return true on success (valid hexadecimal letters); false otherwise. + * + * \note Equivalent to b1 = to_ubyte(c1, c2); b2 = to_ubyte(c3, c4); + */ +static inline bool init_group_no_dash( + int c1, int c2, int c3, int c4, unsigned char& b1, unsigned char& b2 +) { + if (!isxdigit(c1) || !isxdigit(c2) || !isxdigit(c3) || !isxdigit(c4)) { + return false; + } + b1 = to_ubyte(c1, c2); + b2 = to_ubyte(c3, c4); + return true; +} + + +/*! + * \brief Scans an optionally dash-separated hexadecimal + * string into a 16-bytes unique id. + * + * dashes [0] [1] [2] [3] + * 00000000 0000 0000 0000 000000000000 DASH-less format + * 00000000-0000-0000-0000-000000000000 XML UID format + * + * \param s0 pointer to the first character of the string to scan. + * \param end pointer to the one-past-last character of the string to scan. + * \param[out] r pointer to first byte of the 16-bytes uid to initialize. + * \param[out] dashes array of 4 bool indicating which of the 4 possible + * dashes have been seen. + * \return number of dashes seen if the conversion was successful; -1 otherwise. + */ +static inline int init_with_dashes( + const char*const s0, const char*const end, + unsigned char r[16], bool dashes[4] +) { + // no dashes allowed before or in-between first two groups, + // so we can use fixed-offsets into constant s0 pointer. + if (!init_group_no_dash(s0[0], s0[1], s0[2], s0[3], r[0], r[1]) || + !init_group_no_dash(s0[4], s0[5], s0[6], s0[7], r[2], r[3]) + ) { + return -1; + } + + // these 4 statements are correlated, so cannot be reordered. + // but after them, all pointers are constants with fixed offsets, + // so optimizer should have opportunities to reorder/parallelize. + const char*const s1 = (s0[ 8] == '-')? s0 + 1: s0; + const char*const s2 = (s1[12] == '-')? s1 + 1: s1; + const char*const s3 = (s2[16] == '-')? s2 + 1: s2; + const char*const s4 = (s3[20] == '-')? s3 + 1: s3; + + // at this point, we have seen all possible/accepted dashes. So we can + // check the string length against its expected length for a valid uid. + if (s4 + 20 + 12 != end) { + return -1; + } + + // notice each call uses its own sX (possibly offset) pointer (s1, s2, and s3) + if (!init_group_no_dash(s1[ 8], s1[ 9], s1[10], s1[11], r[ 4], r[ 5]) || + !init_group_no_dash(s2[12], s2[13], s2[14], s2[15], r[ 6], r[ 7]) || + !init_group_no_dash(s3[16], s3[17], s3[18], s3[19], r[ 8], r[ 9]) + ) { + return -1; + } + + // no more dashes allowed from this point on, + // so we can use fixed offsets into constant (but possibly offset) s4 pointer. + if (!init_group_no_dash(s4[20], s4[21], s4[22], s4[23], r[10], r[11]) || + !init_group_no_dash(s4[24], s4[25], s4[26], s4[27], r[12], r[13]) || + !init_group_no_dash(s4[28], s4[29], s4[30], s4[31], r[14], r[15]) + ) { + return -1; + } + + // report which dashes we have seen. + dashes[0] = (s1 != s0); + dashes[1] = (s2 != s1); + dashes[2] = (s3 != s2); + dashes[3] = (s4 != s3); + + // and how many dashes we have seen. + return static_cast(s4 - s0); +} + +/*! + * \brief Scans an hexadecimal string into a 16-bytes unique id. + * + * \param s pointer to the 32-hex char string to scan. + * \param[out] r pointer to first byte of the 16-bytes uid to initialize. + * \return true if the conversion was successful; false otherwise. + */ +static inline bool init_no_dash( + const char*const s, unsigned char r[16] +) { + return + init_group_no_dash(s[ 0], s[ 1], s[ 2], s[ 3], r[ 0], r[ 1]) && + init_group_no_dash(s[ 4], s[ 5], s[ 6], s[ 7], r[ 2], r[ 3]) && + init_group_no_dash(s[ 8], s[ 9], s[10], s[11], r[ 4], r[ 5]) && + init_group_no_dash(s[12], s[13], s[14], s[15], r[ 6], r[ 7]) && + init_group_no_dash(s[16], s[17], s[18], s[19], r[ 8], r[ 9]) && + init_group_no_dash(s[20], s[21], s[22], s[23], r[10], r[11]) && + init_group_no_dash(s[24], s[25], s[26], s[27], r[12], r[13]) && + init_group_no_dash(s[28], s[29], s[30], s[31], r[14], r[15]); +} + +/*! + * \brief Parses an hexadecimal string into a 16-bytes array. + * + * \param s pointer to the 32-hex char (or 36 with dashes) string to scan. + * \param len length of \a s string. + * \param[out] r pointer to first byte of the 16-bytes uid to initialize. + * \return true if the parsing was successful; false otherwise. + */ +static inline bool parse_guid( + const char*const s, size_t len, bool lax, unsigned char r[16] +) { + int expected_dashes = 0; + switch (len) { + case 36: ++expected_dashes; [[fallthrough]]; + case 35: ++expected_dashes; [[fallthrough]]; + case 34: ++expected_dashes; [[fallthrough]]; + case 33: ++expected_dashes; + if (!lax && expected_dashes != 4) { + return false; + } else { + bool dashes[4]; + return (expected_dashes == init_with_dashes(s, s + len, r, dashes)); + } + [[fallthrough]]; + case 32: + return init_no_dash(s, r); // try dash-less format + case 0: + std::memset(r, 0, 16); + return true; + } + + return false; +} + +/*! + * \brief Gets a random generator. + * \return the random generator. + */ +#ifndef OES_CORE_UTILS_AVOID_BOOST +static boost::mt19937* randomGenerator() { + static boost::thread_specific_ptr tls; + boost::mt19937* sran = tls.get (); + if (sran == nullptr) { + sran = new boost::mt19937; + // Use random_device result as seed + boost::random_device rd; + sran->seed(rd()); + tls.reset(sran); + } + return sran; +} +#else +static std::mt19937* randomGenerator() { + static thread_local std::unique_ptr tls; + std::mt19937* sran = tls.get(); + if (sran == nullptr) { + sran = new std::mt19937; + // Use random_device result as seed + std::random_device rd; + sran->seed(rd()); + tls.reset(sran); + } + return sran; +} +#endif + +} // namespace + +namespace oes { +namespace core { + +/*! \class Guid oes/core/utils/Guid.h + * \brief %Value class for a Globally Unique Identifier. + * \ingroup oes_core_utils + * + * A Globally Unique Identifier (Guid for short) is a randomly generated + * 128-bit integer. + * + * This class stores this \em integer using a compact representation (16 bytes) + * and allows conversions to (toStringNoDash() and toXmlString()) and from + * (parse()) hexadecimal strings typically used display these guids as text. + * + * A zero-initialized Guid is considered \em invalid. A default-constructed + * Guid is always invalid, as is Guid::invalid() as well. Test a Guid's + * validity using isValid(). + * + * Use the make_guid() factory method to random-initialize a Guid. + * The Guid generator is guaranteed to provide a good random distribution of + * Guids, with virtually no chance of collisions. + * + * Guid is a simple \em value class with trivial copy-construction, + * assignment, and destruction; and is suitable as a key in ordered or + * hashed STL-compliant associative containers. + * + * \sa http://en.wikipedia.org/wiki/Globally_unique_identifier + */ + +/*! + * \typedef unsigned char Guid::value_type + * \brief STL-Compliant typedef for the \em element type of a Guid \em container. + */ + +/*! + * \typedef unsigned char& Guid::reference + * \brief STL-Compliant typedef for a reference to an \em element of a Guid + * \em container. + */ + +/*! + * \typedef unsigned char const& Guid::const_reference + * \brief STL-Compliant typedef for a reference to a const \em element of a + * Guid \em container. + */ + +/*! + * \typedef unsigned char* Guid::iterator + * \brief STL-Compliant typedef for the iterator type of a Guid \em container. + */ + +/*! + * \typedef unsigned char const* Guid::const_iterator + * \brief STL-Compliant typedef for the const_iterator type of a Guid \em container. + */ + +/*! + * \typedef size_t Guid::size_type + * \brief STL-Compliant typedef for the element type of a Guid \em container. + * \brief STL-Compliant typedef for the size-type a Guid \em container. + */ + +/*! + * \brief Creates a new \em invalid Guid. + * + * \note This constructor yields an invalid Guid, equivalent to Guid::invalid(), + * but it is often more expressive, and therefore recommended, to prefer + * using Guid::invalid() explicitly when passed as a argument. + * + * \sa isValid() + * \sa invalid() + */ +Guid::Guid() { + reset(); +} + +/*! + * \brief Generates and returns a new unique identifier. + * \return random Guid. + * + * \sa make_deterministic(const std::string&); + */ +Guid Guid::make() { + /* + Commented out usage of default constructor of random_generator + and go through another constructor for not relying on + memory left uninitialized intentionally. This was causing false + positives in memory checking tool such as valgrind. + See http://www.boost.org/doc/libs/1_46_1/libs/uuid/uuid.html + for additional information. + */ +#ifndef OES_CORE_UTILS_AVOID_BOOST + static boost::thread_specific_ptr> tls; + boost::uuids::basic_random_generator* gen = tls.get(); + if (gen == nullptr) { + gen = new boost::uuids::basic_random_generator (randomGenerator()); + tls.reset (gen); + } +#else + static thread_local std::unique_ptr> tls; + boost::uuids::basic_random_generator* gen = tls.get(); + if (gen == nullptr) { + gen = new boost::uuids::basic_random_generator (randomGenerator()); + tls.reset (gen); + } +#endif + boost::uuids::uuid uid = (*gen)(); + return Guid::from(uid); +} + +/*! + * \brief Creates a new Guid by parsing the input \a uid string. + * + * The string has to be in one of the following format: + * - 00000000000000000000000000000000 (no dash) + * - 00000000-0000-0000-0000-000000000000 (xml convention) + * (where 0 represents an hexadecimal digit, [0-1a-fA-F]). + * + * \param uid the hexadecimal string to parse. + * + * \note Empty strings are considered valid and silently yield an invalid Guid. + * + * \warning If the \a uid string cannot be parsed, this triggers the OES_ASSERT + * which can throw an exception or abort() in debug builds, and yields an + * invalid Guid in release ones. To silently accept unparse-able strings, + * prefer the Guid::parse() factory method. + * + * \sa parse() + * \sa isValid() + */ +Guid::Guid(const String& uid) { + const bool parse_ok = parse_guid(uid.data(), uid.length(), false, guid_); + OES_ASSERT(parse_ok, reset();); +} + +/*! + * \brief Creates a new Guid by parsing the input \a uid string. + * + * The string has to be in one of the following format: + * - 00000000000000000000000000000000 (no dash) + * - 00000000-0000-0000-0000-000000000000 (xml convention) + * (where 0 represents an hexadecimal digit, [0-1a-fA-F]). + * + * \param uid the hexadecimal string to parse. + * + * \note If the \a uid string cannot be parsed, this method just silently + * yields an invalid Guid. + * + * \sa isValid() + * \sa Guid::Guid(const String&) + */ +Guid Guid::parse(const String& uid) { + Guid guid; + if (!parse_guid(uid.data(), uid.length(), false, guid.guid_)) { + guid.reset(); + } + return guid; +} + +/*! + * \brief Assigns this Guid by parsing the input C-style \a str. + * + * The string has to be in one of the following format (if \a lax is false): + * - 00000000000000000000000000000000 (no dash) + * - 00000000-0000-0000-0000-000000000000 (xml convention) + * (where 0 represents an hexadecimal digit, [0-1a-fA-F]). + * + * When \a lax is true, any dash from the xml convention can be omitted, + * provided they remain in their respective relative position. This notably + * allows parsing the 3-dashes format: + * - 00000000-00000000-0000-000000000000 + * + * \param str the Guid string to parse. + * \param len the length of the hexadecimal (and optional dashes) Guid string. + * \param lax whether to allow optional dashes or not. + * \return true on a successful parse; false otherwise, leaving this Guid in a + * partially parsed state. Caller should reset() this Guid if it matters. + * + * \note Guid::parse() does not allow to distinguish an invalid-format guid, + * from a valid-format \em invalid Guid (i.e. the \em null zero-only Guid). + * While this method does. + * + * \note This method avoids creating a throw-away std::string, just to parse it, + * unlike Guid::parse(). Especially since Guids exceed std::string's SBO's + * Small Buffer Optimization threshold, so always results in heap allocation. + * + * \warning On unsuccessful parses, this Guid may have been partially updated. + * Caller should take care to explicitly reset() this Guid, if it matters. + * + * \sa isValid() + * \sa Guid::parse(const String&) + */ +bool Guid::reparse(const char* str, size_t len, bool lax) { + return parse_guid(str, len, lax, guid_); +} + +/*! + * \brief Creates a copy of another \em foreign Guid. + * + * This constructor allows to efficiently convert another byte-array-based + * Guid-like type into an instance of this Guid type, without going through + * an intermediate string representation. + * + * Both \a begin and \a end should \em random const iterators to bytes + * (unsigned char), and the \a distance between them must be exactly 16. + * + * \note this method is typically not used directly, but via the Guid::from() + * template helper. + * + * \param begin start of byte array from foreign guid type. + * \param end end of byte array started at \a end. + */ +Guid::Guid(const_iterator begin, const_iterator end) { + OES_ASSERT(end - begin == 16, reset(); return;); + std::copy(begin, begin + size(), guid_); +} + +/*! + * \brief Gets the static invalid Guid. + * \return the const invalid guid. + */ +const Guid& Guid::invalid() { + static const Guid invalid_uid(zeros__, zeros__ + 16); + return invalid_uid; +} + +/*! + * \brief Namespace for a fully-qualified domain name. + * \return RFC 4122 Appendic C defined known value. + */ +Guid Guid::dns_namespace() { + static const unsigned char bytes[] = { + 0x6b, 0xa7, 0xb8, 0x10, 0x9d, 0xad, 0x11, 0xd1, + 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8 + }; + return Guid(bytes, bytes + 16); +} + +/*! + * \brief Namespace for a URL. + * \return RFC 4122 Appendic C defined known value. + */ +Guid Guid::url_namespace() { + static const unsigned char bytes[] = { + 0x6b, 0xa7, 0xb8, 0x11, 0x9d, 0xad, 0x11, 0xd1, + 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8 + }; + return Guid(bytes, bytes + 16); +} + +/*! + * \brief Namespace for an ISO OID. + * \return RFC 4122 Appendic C defined known value. + */ +Guid Guid::oid_namespace() { + static const unsigned char bytes[] = { + 0x6b, 0xa7, 0xb8, 0x12, 0x9d, 0xad, 0x11, 0xd1, + 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8 + }; + return Guid(bytes, bytes + 16); +} + +/*! + * \brief Namespace for an X.500 DN (in DER or a text output format). + * \return RFC 4122 Appendic C defined known value. + */ +Guid Guid::x500_namespace() { + static const unsigned char bytes[] = { + 0x6b, 0xa7, 0xb8, 0x14, 0x9d, 0xad, 0x11, 0xd1, + 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8 + }; + return Guid(bytes, bytes + 16); +} + +/*! + * \brief Default \em named Guid namespace specific to Open ETP Server. + * \return Fixed (arbitrary) value defining the \em Open ETP Server namespace. + */ +Guid Guid::oes_namespace() { + // SHA-1 key f3c92614d98d1383c5b23d4e70ea197ac5ee51b0 + static const unsigned char bytes[] = { + 0xf3, 0xc9, 0x26, 0x14, 0xd9, 0x8d, 0x13, 0x83, + 0xc5, 0xb2, 0x3d, 0x4e, 0x70, 0xea, 0x19, 0x7a + }; + return Guid(bytes, bytes + 16); +} + +/*! + * \fn Guid Guid::make_deterministic(const std::string& arbitrary_text) + * \brief Convenience method defaulting the namespace to oes_namespace(). + * \param arbitrary_text the text to compute the Guid from. + * \return A \em name-based non-random Guid, based on this calls input. + * \note the same input always yield the same Guid value. + */ + +/*! + * \brief Generates and returns a new unique identifier, deterministically, + * based on the given namespace and text. + * \param namespace_guid the \em namespace (seed) for the generation. + * \param arbitrary_text the text to compute the Guid from. + * \return A \em name-based non-random Guid, based on this calls input. + * \note the same input always yield the same Guid value. + */ +Guid Guid::make_deterministic(const Guid& namespace_guid, const std::string& arbitrary_text) { + boost::uuids::uuid namespace_uuid; + std::copy(namespace_guid.begin(), namespace_guid.end(), namespace_uuid.data); + boost::uuids::name_generator gen(namespace_uuid); + boost::uuids::uuid result(gen(arbitrary_text)); + return Guid::from(result); +} + +/*! + * \brief Convenience method taking the namespace as text. + * \param namespace_text the \em namespace (seed) for the generation, which can + * be either a valid string-representation of a Guid, in which case it is parsed + * as a Guid and used as-is for the namespace, or arbitrary text, in which case + * that text is first converted to a Guid itself, using make_deterministic() and + * the default oes_namespace(), to be used as the namespace. + * \param arbitrary_text the text to compute the Guid from. + * \return A \em name-based non-random Guid, based on this calls input. + * \note the same input always yield the same Guid value. + */ +Guid Guid::make_deterministic(const std::string& namespace_text, const std::string& arbitrary_text) { + Guid ns = Guid::parse(namespace_text); + if (!ns.isValid()) { + ns = make_deterministic(namespace_text); + } + return make_deterministic(ns, arbitrary_text); +} + +/*! + * \fn bool Guid::isValid() const + * \brief Determines whether this Guid is valid. + * \return true if this Guid is valid; false otherwise. + */ + +/*! + * \fn Guid::iterator Guid::begin() + * \brief Gets STL-compliant range-starting iterator to internal byte array. + * \return the iterator to start of 16-bytes long array. + */ + +/*! + * \fn Guid::const_iterator Guid::begin() const + * \brief Gets STL-compliant range-starting iterator to internal byte array. + * \return the const_iterator to start of 16-bytes long array. + */ + +/*! + * \fn Guid::iterator Guid::end() + * \brief Gets STL-compliant range-ending iterator to internal byte array. + * \return the iterator to the end (one-past last byte) of 16-bytes long array. + */ + +/*! + * \fn Guid::const_iterator Guid::end() const + * \brief Gets STL-compliant range-ending iterator to internal byte array. + * \return the iterator to the end (one-past last byte) of 16-bytes long array. + */ + +/*! + * \fn Guid::size_type Guid::size() const + * \brief Gets the length of the byte array internal to this Guid. + * \return Always 16. + */ + +/*! + * \fn void Guid::reset() + * \brief Resets this Guid to invalid. + * \sa isValid() + */ + +/*! + * \brief Converts this Guid to a dash-less hexadecimal string. + * + * Examples: + * - 00000000000000000000000000000000 (for invalid Guid instances) + * - 6f2fe04a3ea7490fa7d7469329bf3045 (for a given valid Guid) + * + * \return a 32 character long hexadecimal string. + */ +String Guid::toStringNoDash() const { + const char characters[] = { // 32 hex chars + to_char(guid_[ 0] >> 4), to_char(guid_[ 0] & 0xf), + to_char(guid_[ 1] >> 4), to_char(guid_[ 1] & 0xf), + to_char(guid_[ 2] >> 4), to_char(guid_[ 2] & 0xf), + to_char(guid_[ 3] >> 4), to_char(guid_[ 3] & 0xf), + to_char(guid_[ 4] >> 4), to_char(guid_[ 4] & 0xf), + to_char(guid_[ 5] >> 4), to_char(guid_[ 5] & 0xf), + to_char(guid_[ 6] >> 4), to_char(guid_[ 6] & 0xf), + to_char(guid_[ 7] >> 4), to_char(guid_[ 7] & 0xf), + to_char(guid_[ 8] >> 4), to_char(guid_[ 8] & 0xf), + to_char(guid_[ 9] >> 4), to_char(guid_[ 9] & 0xf), + to_char(guid_[10] >> 4), to_char(guid_[10] & 0xf), + to_char(guid_[11] >> 4), to_char(guid_[11] & 0xf), + to_char(guid_[12] >> 4), to_char(guid_[12] & 0xf), + to_char(guid_[13] >> 4), to_char(guid_[13] & 0xf), + to_char(guid_[14] >> 4), to_char(guid_[14] & 0xf), + to_char(guid_[15] >> 4), to_char(guid_[15] & 0xf) + }; + String string_guid(characters, characters + 32); + return string_guid; +} + +/*! + * \brief Converts this Guid to a dash-separated hexadecimal string. + * + * This particular representation is common to many XML schemas, and is used + * as the \em standard representation for the streaming operator. + * + * Examples: + * - 00000000-0000-0000-0000-000000000000 (for invalid Guid instances) + * - 6f2fe04a-3ea7-490f-a7d7-469329bf3045 (for a given valid Guid) + * + * \return a 36 character long hexadecimal string. + */ +String Guid::toXmlString() const { + static const char dash ('-'); + const char characters[] = { // 32 hex chars + 4 dash separators + to_char(guid_[ 0] >> 4), to_char(guid_[ 0] & 0xf), + to_char(guid_[ 1] >> 4), to_char(guid_[ 1] & 0xf), + to_char(guid_[ 2] >> 4), to_char(guid_[ 2] & 0xf), + to_char(guid_[ 3] >> 4), to_char(guid_[ 3] & 0xf), + dash, // 8 + to_char(guid_[ 4] >> 4), to_char(guid_[ 4] & 0xf), + to_char(guid_[ 5] >> 4), to_char(guid_[ 5] & 0xf), + dash, // 13 + to_char(guid_[ 6] >> 4), to_char(guid_[ 6] & 0xf), + to_char(guid_[ 7] >> 4), to_char(guid_[ 7] & 0xf), + dash, // 18 + to_char(guid_[ 8] >> 4), to_char(guid_[ 8] & 0xf), + to_char(guid_[ 9] >> 4), to_char(guid_[ 9] & 0xf), + dash, // 23 + to_char(guid_[10] >> 4), to_char(guid_[10] & 0xf), + to_char(guid_[11] >> 4), to_char(guid_[11] & 0xf), + to_char(guid_[12] >> 4), to_char(guid_[12] & 0xf), + to_char(guid_[13] >> 4), to_char(guid_[13] & 0xf), + to_char(guid_[14] >> 4), to_char(guid_[14] & 0xf), + to_char(guid_[15] >> 4), to_char(guid_[15] & 0xf) + }; + + std::string string_guid(characters, characters + 36); + return string_guid; +} + +/*! + * \brief Swaps the contents of two Guids. + * \param lhs the left-hand-side Guid to swap. + * \param rhs the right-hand-side Guid to swap. + */ +void swap(oes::core::Guid& lhs, oes::core::Guid& rhs) { + std::swap_ranges(lhs.begin(), lhs.end(), rhs.begin()); +} + +/*! + * \brief Gets the hash value of this Guid, for hashing containers. + * \param uid the Guid to get the hash of. + * \return the hash-value. + */ +std::size_t hash_value(const oes::core::Guid& uid) { + // Marginally faster, but makes hash endianness dependent. + //return *reinterpret_cast(uid.begin()); + return boost::hash_range(uid.begin(), uid.end()); +} + +/*! \fn template Guid Guid::from(const T& guid) + * \brief Creates a Guid as a copy of another \em foreign Guid. + * \tparam T foreign Guid-like type with begin()/end() methods to + * extract the range to the 6-byte array storing the foreign Guid. + * \param guid the foreign guid to copy. + * \return the guid. + */ + +/*! \fn Guid make_guid() + * \brief Makes (generates) a randomly initialized Guid. + * + * Helper allowing to write make_guid() instead of Guid::make(), + * for increased code readability. + * + * This method can also be used to fill a container with random guids: + * \code + * std::vector guids(1000); + * std::generate(guids.begin(), guids.end(), &make_guid); + * \endcode + * + * \return valid Guid. + */ + +/*! \fn int Guid::compare(const Guid&) const + * \brief Compares this guid with another. + * + * The comparison is consistent with how std::memcmp behaves. + * + * \param other the other guid to compare against. + * \return 0 if equal; + * negative integer if less-than other; + * positive integer otherwise. + */ + +/*! \fn bool operator==(const Guid& lhs, const Guid& rhs) + * \brief Tests two guids for equality. + * \param lhs the left-hand-side operand. + * \param rhs the right-hand-side operand. + * \return true if \a lhs equals \a rhs; false otherwise. + */ + +/*! \fn bool operator!=(const Guid& lhs, const Guid& rhs) + * \brief Tests two guids for inequality. + * \param lhs the left-hand-side operand. + * \param rhs the right-hand-side operand. + * \return true if \a lhs does \b not equal \a rhs; false otherwise. + */ + +/*! \fn bool operator<(const Guid& lhs, const Guid& rhs) + * \brief Tests whether one Guid is strictly lesser than another guid. + * + * Implements a strict weak ordering for Guid%s, suitable for STL-compliant + * ordered containers. + * + * \param lhs the left-hand-side operand. + * \param rhs the right-hand-side operand. + * \return true if \a lhs is less than \a rhs, i.e. orders before it; + * false otherwise. + */ + +/*! \fn bool operator>(const Guid& lhs, const Guid& rhs) + * \brief Tests whether one Guid is strictly greater than another guid. + * \param lhs the left-hand-side operand. + * \param rhs the right-hand-side operand. + * \return true if \a lhs is greater than \a rhs; false otherwise. + */ + +/*! \fn bool operator<=(const Guid& lhs, const Guid& rhs) + * \brief Tests whether one Guid is lesser than or equal another guid. + * \param lhs the left-hand-side operand. + * \param rhs the right-hand-side operand. + * \return true if TODO; false otherwise. + * \return true if \a lhs is lesser than or equal to \a rhs; false otherwise. + */ + +/*! \fn bool operator>=(const Guid& lhs, const Guid& rhs) + * \brief Tests whether one Guid is greater than or equal another guid. + * \param lhs the left-hand-side operand. + * \param rhs the right-hand-side operand. + * \return true if TODO; false otherwise. + * \return true if \a lhs is greater than or equal to \a rhs; false otherwise. + */ + +/*! \fn std::ostream& operator<<(std::ostream& os, const Guid& uid) + * \brief Outputs a Guid to a stream, using the dash-separated format. + * \param[in, out] os the stream to output to. + * \param uid the Guid to output. + * \return the input \a os, for call chaining. + * \sa Guid::toXmlString() + */ + +} // namespace core +} // namespace oes diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.h new file mode 100644 index 0000000000000000000000000000000000000000..479e9934ba0134b937734e5ec35ef5ed04267435 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.h @@ -0,0 +1,208 @@ +// ============================================================================ +// Copyright 2013-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_TYPES_CORE_GUID_H +#define OES_CORE_TYPES_CORE_GUID_H + +#include +#include + +#include +#include +#include + +// When compiling CUDA source on Intel 15 and earlier platforms, +// certain C++11 experimental features need to be excluded from the compilation. +// In this case, unordered_map include breaks CUDA compilation. +#if (defined PG_SYS_OS_Linux) && !(defined __GXX_EXPERIMENTAL_CXX0X__) +# define OES_OLD_CUDA_PLATFORM +#endif + +#if !(defined OES_OLD_CUDA_PLATFORM) +# include +# include +# include +#endif + +namespace oes { +namespace core { + +class OES_CORE_UTILS_EXPORT Guid { +public: + typedef unsigned char value_type; + typedef unsigned char& reference; + typedef unsigned char const& const_reference; + typedef unsigned char* iterator; + typedef unsigned char const* const_iterator; + typedef size_t size_type; + +public: + Guid(); + explicit Guid(const String& uid); + Guid(const_iterator begin, const_iterator end); + + // ~Guid() = default; + // Guid(const Guid&) = default; + // Guid& operator=(const Guid&) = default; + + template + static Guid from(const T& uid) { + const auto first = uid.begin(); + return Guid(&*first, (&*first)+std::distance(first, uid.end())); + } + + static Guid make(); + static Guid parse(const String& uid_string); + static const Guid& invalid(); + static size_type static_size() { + return 16; + } + + static Guid dns_namespace(); + static Guid url_namespace(); + static Guid oid_namespace(); + static Guid x500_namespace(); + static Guid oes_namespace(); // default namespace + static Guid make_deterministic(const std::string& arbitrary_text); + static Guid make_deterministic(const Guid& namespace_guid, const std::string& arbitrary_text); + static Guid make_deterministic(const std::string& namespace_text, const std::string& arbitrary_text); + + // explicit operator bool() const; + bool isValid() const; + size_type size() const; + const_iterator begin() const; + const_iterator end() const; + + String toXmlString() const; + String toStringNoDash() const; + + bool reparse(const char* str, size_t len, bool lax); + + void reset(); + iterator begin(); + iterator end(); + + int compare(const Guid& other) const; + +private: + unsigned char guid_[16]; +}; + + +inline Guid make_guid() { + return Guid::make(); +} + +OES_CORE_UTILS_EXPORT void swap(Guid& lhs, Guid& rhs); +OES_CORE_UTILS_EXPORT std::size_t hash_value(const Guid& uid); + +#if !(defined OES_OLD_CUDA_PLATFORM) + +// type definitions +using SetOfGuid = std::set ; +using VectorOfGuid = std::vector ; + +struct GuidHashFunctor { + std::size_t operator()(const Guid& guid) const { + return (hash_value (guid)); + } +}; + +template +using UnorderedMapOfGuid = std::unordered_map < Guid, VALUE_TYPE, GuidHashFunctor>; + +#else +# undef OES_OLD_CUDA_PLATFORM +#endif + +// inline Guid::operator bool() const { +// return isValid(); +// } + +inline bool Guid::isValid() const { + const unsigned char*const d = guid_; + return + d[0x0] || d[0x1] || d[0x2] || d[0x3] || d[0x4] || d[0x5] || d[0x6] || d[0x7] || + d[0x8] || d[0x9] || d[0xa] || d[0xb] || d[0xc] || d[0xd] || d[0xe] || d[0xf]; +} + +inline Guid Guid::make_deterministic(const std::string& arbitrary_text) { + return make_deterministic(oes_namespace(), arbitrary_text); +} + +inline Guid::iterator Guid::begin() { + return guid_; +} + +inline Guid::const_iterator Guid::begin() const { + return guid_; +} + +inline Guid::iterator Guid::end() { + return guid_ + static_size(); +} + +inline Guid::const_iterator Guid::end() const { + return guid_ + static_size(); +} + +inline Guid::size_type Guid::size() const { + return static_size(); +} + +inline void Guid::reset() { + std::memset(guid_, 0, static_size()); // zero-init, thus isValid() == false +} + +inline int Guid::compare(const Guid& other) const { + if (*guid_ == *other.guid_) { + return std::memcmp(guid_, other.guid_, static_size()); + } + return (*guid_ < *other.guid_)? -1: +1; +} + +inline bool operator==(const Guid& lhs, const Guid& rhs) { + return 0 == lhs.compare(rhs); +} + +inline bool operator!=(const Guid& lhs, const Guid& rhs) { + return 0 != lhs.compare(rhs); +} + +inline bool operator<(const Guid& lhs, const Guid& rhs) { + return lhs.compare(rhs) < 0; +} + +inline bool operator>(const Guid& lhs, const Guid& rhs) { + return lhs.compare(rhs) > 0; +} + +inline bool operator<=(const Guid& lhs, const Guid& rhs) { + return lhs.compare(rhs) <= 0; +} + +inline bool operator>=(const Guid& lhs, const Guid& rhs) { + return lhs.compare(rhs) >= 0; +} + +inline std::ostream& operator<<(std::ostream& os, const Guid& uid) { + return os << uid.toXmlString(); +} + +} // namespace core +} // namespace oes + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.hpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.hpp new file mode 100644 index 0000000000000000000000000000000000000000..ce283e0c99e4a1e9e103b8ba1b86c8942ae646ef --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Guid.hpp @@ -0,0 +1,44 @@ +// ============================================================================ +// Copyright 2015-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_TYPES_CORE_GUID_HPP +#define OES_CORE_TYPES_CORE_GUID_HPP + +#include + +#include // for std::hash +#include + +namespace std { + +template <> struct hash { + typedef oes::core::Guid argument_type; + typedef std::size_t result_type; + + result_type operator()(argument_type const& s) const { + return oes::core::hash_value(s); + } +}; + +} // namespace std + +namespace oes { +namespace core { + using GuidHashSet = std::unordered_set; +} // namespace core +} // namespace oes + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3f48d5fbb0357407c97fd3f60271e2d60156b653 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.cpp @@ -0,0 +1,4062 @@ +// ============================================================================ +// Copyright 2013-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +/*! \file oes/core/utils/Json.h + * \brief Helper classes to generate and parse well formatted JSON document. + * + * Use JSONWriter to build a JSON document. + * Use json_parse() with a custom JSONHandler to interpret the structure + * of a JSON document. + */ + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +/*! \internal + * \brief Array of JSON type strings. + * \warning Elements must be declared in the same order as in the JSONType enum + */ +const std::string json_type_strings[] = { + "", + "object", + "array", + "string", + "number", + "boolean" + // note no entry for JSON_ANY +}; + +const size_t json_type_count = sizeof(json_type_strings) / sizeof(*json_type_strings); + +/*! + * \internal + * \brief Determines whether a character is JSON insignificant whitespace. + * + * \sa http://tools.ietf.org/html/rfc4627, 2., insignificant whitespace. + * \note that this differs from isspace, which adds Vertical Tab 0x0B and + * Form Feed 0x0C, to Space 0x20, Horizontal Tab 0x09, New Line / + * Line Feed 0x0A, and Carriage Return 0x0D. + */ +inline bool is_json_whitespace(unsigned char c) { + return c == 0x20 || c == 0x09 || c == 0x0A || c == 0x0D; +} + +/*! + * \internal + * \brief Encodes arbitrary string into a JSON string in double-quotes. + * + * Special single-byte characters like ", \, /, \b, \f, \n, \r, and \t are + * replaced with their classic two-bytes C-literal equivalent. + * + * Unicode codepoints encoded on multiple bytes are left unchanged. + * Non-printable 7-bit ASCII characters are converted to \u00FF. + * + * \param os the stream to write the encoded string to. + * \param str UTF-8 encoded string. + */ +void encode_json_string(std::ostream& os, const std::string& str) { + os << '"'; // opening double-quote + const size_t len = str.length(); + for (size_t n = 0; n < len; ++n) { + const unsigned char c = str[n]; + if (c > 127) { + // All multi-byte UTF-8 encoded codepoints are > 127 return + // them as-is for now instead of converting them to \uFFFF + // (or a \uFFFF\uFFFF surrogate pair above the BMP) for now. + os.put(c); + continue; + } + switch (c) { + // double-quote, reverse solidus, and control characters + case '"': os.put('\\'); os.put('"'); break; + case '\\': os.put('\\'); os.put('\\'); break; + case '/': os.put('\\'); os.put('/'); break; + case '\b': os.put('\\'); os.put('b'); break; + case '\f': os.put('\\'); os.put('f'); break; + case '\n': os.put('\\'); os.put('n'); break; + case '\r': os.put('\\'); os.put('r'); break; + case '\t': os.put('\\'); os.put('t'); break; + default: + if (isprint(c)) { + os.put(c); + } else { + os.put('\\'); + os.put('u'); + os << std::setw(4) << std::setfill('0') // not "sticky" + << std::hex << (((unsigned int)c) & 0xFF) + << std::dec; // reset to decimal! std::hex is "sticky"! + } + } + } + os << '"'; // closing double-quote +} + +/*! + * \internal + * \brief Custom stream buffer wrapped around a memory region. + * \sa http://goo.gl/ZmUWl + */ +struct imembuf : public std::streambuf { + imembuf(const char* beg, const char* end) { + char* s = const_cast(beg); + setg(s, s, const_cast(end)); + } +}; + +template +T parse_number(const char*const s, const char*const end) { + // avoid std::string (heap-alloc+copy) and another copy in istringstream + // by using a one-time stack-allocated custom stream-buffer + istream. + //std::string buf(s, end); + //std::istringstream is(buf); + imembuf rd(s, end); + std::istream is(&rd); + T value = 0; // use uint64_t instead? + is >> value; + if (is.fail()) { // handles integer overflow + throw std::runtime_error("Invalid number"); + } + return value; +} + +uint64_t parser_integer(const char*& s, const char*const end) { + // skip leading zeros (to get the magnitude right). + const char*const prev_s0 = s; + for (unsigned char c = *s; s < end && c == '0'; c = *++s); + + const char*const prev_s = s; + for (unsigned char c = *s; s < end; c = *++s) { + if (c < '0' || '9' < c) { + break; + } + } + const size_t magnitude10 = s - prev_s; + if (magnitude10 == 0) { + if (prev_s0 < prev_s) { + return 0; // one-or-more zero(s) + } + throw std::runtime_error("Invalid number: Expected digits"); + } + + // Max 64-bit size_t 18,446,744,073,709,551,615 (magnitude10 = 20), + // so anything larger will not fit into a size_t / uint64_t. + if (magnitude10 > 20) { + // (if a bunch of trailing zeros, might fit into a double...) + throw std::runtime_error("Invalid number: overflows uint64_t"); + } + + // if it easily fits into a 32-bit int (below one billion), + // use the cheaper atoi conversion (Max 32-bit size_t 4,294,967,295). + if (magnitude10 < 10) { + char buf[10] = { 0 }; // null-terminated + strncpy(buf, prev_s, magnitude10); + return atoi(buf); + } + + // pay for the more expensive istream::operator>> conversion, + // which will detect/handle integer overflow, on which we throw. + return parse_number(prev_s, s); +} + +} // namespace + +namespace oes { +namespace core { + +/*! \class JSONWriterOptions oes/core/utils/Json.h + * \brief Options for JSONWriter. + */ + +/*! \var JSONWriterOptions::version_ + * Indicates the version number of the JSon document to write. + */ + +/*! \var JSONWriterOptions::omit_defaults_ + * For values supporting this option (typically serialized as objects), + * do not serialize or output anything for fields who's value is that + * fields default value. Taking this JSONWriterOptions class as an example + * the default created instance would serialize as just {}, and one + * where omit_defaults_ is true would serialize as {"omit-defaults":true}, + * when this option is true. + * + * Defaults to false. + * + * \note this is only a hint, and JSON serialization code is free to ignore it. + */ + +/*! \var JSONWriterOptions::null_on_empty_ + * For values supporting this option, for fields of types object, array, + * or string, use null rather than the \em empty representation for that + * field value, i.e. {}, [], and "" respectively, when this option is true. + * + * Defaults to false. + * + * \note this is only a hint, and JSON serialization code is free to ignore it. + */ + +/*! \class JSONWriter oes/core/utils/Json.h + * \brief Builds a JSON document. + * + * JavaScript Object Notation (JSON) is a text format for the + * serialization of structured data. It is derived from the object + * literals of JavaScript, as defined in the ECMAScript Programming + * Language Standard, Third Edition. + * + * JSON can represent four primitive types (strings, numbers, booleans, + * and null) and two structured types (objects and arrays). + * + * A string is a sequence of zero or more Unicode characters [UNICODE]. + * + * An object is an unordered collection of zero or more name/value + * pairs, where a name is a string and a value is a string, number, + * boolean, null, object, or array. + * + * An array is an ordered sequence of zero or more values. + * + * JSONWriter will write out a well-formed JSON document (which is also a + * value) to a given stream. No decorations like spaces or line-separators + * are added to the output text, which thus is composed of a single line. + * + * For example, the calls below result in the text {"version":"1.2.3","values":[1,2.3,true,null,{}]}. + * \code + * void print_json() { + * JSONWriter json(std::cout); + * json.push_object() + * .key("version").string("1.2.3") + * .key("values").push_array() + * .number(1).number(2.3).json_true().json_null() + * .push_object().pop_object() + * .pop_array() + * .pop_object(); + * } + * \endcode + * + * \note When instead of writing the JSON text to a std::ostream, you want to + * directly get the JSON text itself, use the to_json_text() convenience + * template function (defined in json.hpp) which + * encapsulates the 4 lines of code necessary to do just that. + * + * \note This writer allows writing a single \em primitive value (either a + * number, a string, a boolean, or null) by itself, outside of an array + * or an object. + * + * \sa http://tools.ietf.org/html/rfc4627 + */ + +/*! + * Constructs a new JSONWriter to write around a given ostream. + * \param os Stream to write to. Must remain valid during this writer's lifetime. + * \param options the writing options. + * + * \warning floating-point values will be written using that streams formatting + * rules, which may result in truncation of some decimals. You may want to + * call os.precision(17) (for example) to increase the default precision + * used, and/or os.setf(ios::fixed) to force fixed-precision. + */ +JSONWriter::JSONWriter( + std::ostream& os, const JSONWriterOptions& options +) : os_(os) + , options_(options) +{ +} + +/*! + * \brief Deletes this writer. + * + * \throw std::uncaught_exception on an incomplete document, if not reset(), + * unless std::uncaught_exception() returns true, in which case the + * exception is not thrown to avoid std::terminate(). + * + * \warning If the state of this writer is not ok(), i.e. an invalid JSON + * value results from the series of calls made on this writer, then this + * destructor throws (unless another exception is pending to avoid + * std::terminate()ing the application). Use reset() prior to destruction + * to avoid this possible exception if appropriate. + */ +#if (__cplusplus > 199711L) || (defined (_MSVC_LANG) && (_MSVC_LANG >= 201402L)) +JSONWriter::~JSONWriter() noexcept(false) { +#else + JSONWriter::~JSONWriter() { +#endif +#if (__cplusplus >= 201703L) || (defined (_MSVC_LANG) && (_MSVC_LANG >= 201703L)) + if (!ok () && (std::uncaught_exceptions () == 0)) { +#else + if (!ok() && !std::uncaught_exception()) { +#endif + // cppcheck-suppress exceptThrowInDestructor + throw std::runtime_error("Invalid or incomplete JSON value"); + } +} + +/*! + * \brief Determines whether we have written a valid and complete JSON value. + * \return true if valid and complete; false otherwise. + * \sa reset() + */ +bool JSONWriter::ok() const { + return end_.empty() || (end_.size() == 1 && end_[0] == ','); +} + +/*! + * \brief Resets the state of this class. + * \note After a reset(), the destruction (implicit or not) of this writer + * is guaranteed not to throw. + * \warning the output-stream this writer wraps is left unchanged, with whatever + * content was written so far already \em consumed by that stream. + * \sa ok() + */ +void JSONWriter::reset() { + end_.clear(); + pending_ws_.clear(); +} + +/*! + * \brief gets the writing options. + */ +const JSONWriterOptions& JSONWriter::options() const { + return options_; +} + +/*! + * \internal + * \brief Asserts that we expect an object key. + */ +void JSONWriter::assert_expects_key() { + if (end_.empty()) { + throw std::runtime_error("Unexpected top-level object key"); + } + + switch (end_.back()) { + case ']': + throw std::runtime_error("Expecting a value, not an object key"); + case ':': + throw std::runtime_error("Expecting a value, following an object key"); + case ',': + if (end_.size() == 1) { + throw std::runtime_error("Unexpected top-level object key"); + } + if (end_[end_.size()-2] != '}') { + throw std::runtime_error("Unexpected object key outside object"); + } + os_.put(','); + end_.pop_back(); + break; + } + flush_pending_ws(); +} + +/*! + * \internal + * \brief Asserts that we expect a value. + * Could be a naked value, a value of an array, or the value of a given object key. + */ +void JSONWriter::assert_expects_value() { + // accept "naked" values, not only object as the top-level value. + if (end_.empty()) { + //flush_pending_ws(); + return; + } + + switch (end_.back()) { + case '}': + throw std::runtime_error("Expecting an object key, not a value"); + case ':': + BOOST_ASSERT(end_.size() >= 2); + os_.put(':'); + end_.pop_back(); + break; + case ',': + if (end_.size() == 1) { + throw std::runtime_error("Unexpected top-level value"); + } + if (end_[end_.size()-2] == '}') { + throw std::runtime_error("Expecting an object key, not a value"); + } + os_.put(','); + end_.pop_back(); + break; + } + flush_pending_ws(); +} + +/*! + * \internal + * \brief Asserts that we expect a value, and writes that value. + */ +template JSONWriter& JSONWriter::push_value(T t) { + assert_expects_value(); + os_ << t; + end_.push_back(','); + return *this; +} + +/*! + * \brief Appends an integer value to the JSON document. + * + * \param value the integer to append. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if an primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::number(int value) { + return push_value(value); +} + +/*! + * \brief Appends an integer value to the JSON document. + * + * \param value the integer to append. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if an primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::number(uint64_t value) { + return push_value(value); +} + +/*! + * \brief Appends an unsigned integer value to the JSON document. + * + * \param value the unsigned integer to append. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if an primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::number(unsigned int value) { + return push_value(value); +} + +#ifdef _WIN32 +/*! + * \brief Appends an unsigned long value to the JSON document. + * + * \param value the unsigned long to append. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if an primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::number(unsigned long value) { + return push_value(value); +} +#endif + +/*! + * \brief Appends an integer value to the JSON document. + * + * \param value the integer to append. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if an primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::number(int64_t value) { + return push_value(value); +} + +/*! + * \brief Appends a floating-point value to the JSON document. + * + * \param value the floating-point to append. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if a primitive value is not allowed, + * typically because an object key is expected instead. + * + * \warning Since JSON is a textual document, and we write it via a + * std::ostream, that stream's current formatting rules are used as-is, + * in particular its std::ostream::precision() and formatting flags, + * which could result in loss of precision of \a value when read back. + * Client is in charge of configuring its stream suitably. + */ +JSONWriter& JSONWriter::number(float value) { + return push_value(value); +} + +/*! + * \brief Appends a floating-point value to the JSON document. + * + * \param value the floating-point to append. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if a primitive value is not allowed, + * typically because an object key is expected instead. + * + * \warning Since JSON is a textual document, and we write it via a + * std::ostream, that stream's current formatting rules are used as-is, + * in particular its std::ostream::precision() and formatting flags, + * which could result in loss of precision of \a value when read back. + * Client is in charge of configuring its stream suitably. + */ +JSONWriter& JSONWriter::number(double value) { + return push_value(value); +} + +/*! + * \brief Appends a string value to the JSON document. + * + * Special single-byte characters like ", \\, /, \\b, \\f, \\n, \\r, and \\t are + * replaced with their classic two-bytes C-literal equivalent. + * + * Unicode codepoints UTF-8 encoded on multiple bytes are left unchanged. + * Non-printable 7-bit ASCII characters are converted to \\u00FF (where + * FF denotes that non-printable character hexadecimal ASCII code). + * + * \param value the string to append. + * \param null_on_empty (Optional; defaults to false) whether to write an empty + * string (""), or null in case of an empty string. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if a primitive value is not allowed, + * typically because an object key is expected instead. + * + * \note One should use key(), not string() to setup the key of an + * object's key-value pair. + */ +JSONWriter& JSONWriter::string(const std::string& value, bool null_on_empty) { + assert_expects_value(); + if (null_on_empty && value.empty()) { + os_ << "null"; + } else { + encode_json_string(os_, value); + } + end_.push_back(','); + return *this; +} + +/*! + * \brief Appends the literal "true" or "false" to the JSON document. + * + * \param value if \a value is true, appends "true"; "false" otherwise. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if a primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::json_bool(bool value) { + return value? json_true(): json_false(); +} + +/*! + * \brief Appends the literal "null" to the JSON document. + * + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if a primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::json_null() { + return push_value("null"); +} + +/*! + * \brief Appends the literal "true" to the JSON document. + * + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if a primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::json_true() { + return push_value("true"); +} + +/*! + * \brief Appends the literal "false" to the JSON document. + * + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if a primitive value is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::json_false() { + return push_value("false"); +} + +/*! + * \brief Starts / opens / pushes a new array in the JSON document. + * + * A JSON array is an ordered sequence of zero or more values. + * + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if an array is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::push_array() { + assert_expects_value(); + os_ << '['; + end_.push_back(']'); + return *this; +} + +/*! + * \brief Finishes / closes / pops the current array of the JSON document. + * + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if the current \em scope is not + * inside an array. + */ +JSONWriter& JSONWriter::pop_array() { + if (end_.empty()) { + throw std::runtime_error("No open array to end"); + } + if (end_.back() == ',') end_.pop_back(); // discard + if (end_.empty() || end_.back() != ']') { + throw std::runtime_error("No open array in this scope"); + } + flush_pending_ws(); + os_ << end_.back(); + end_.pop_back(); + end_.push_back(','); + return *this; +} + +/*! + * \brief Starts / opens / pushes a new object in the JSON document. + * + * A JSON object is an unordered collection of zero or more name/value + * pairs, where a name is a string and a value is a string, number, + * boolean, null, object, or array. + * + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if an object is not allowed, + * typically because an object key is expected instead. + */ +JSONWriter& JSONWriter::push_object() { + assert_expects_value(); + os_ << '{'; + end_.push_back('}'); + return *this; +} + +/*! + * \brief Sets the key name of an object's key-value pair. + * + * The key must be followed by a primitive value (number(), string(), + * json_bool(), json_null()), or an array (push_array()) or object (push_object()). + * Failure to provide this value is likely to make this writer throw an + * exception in its destructor, unless reset() is called prior to this writer + * going out of scope. + * + * \param key the key-value pair's name. + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if the current \em scope is not + * inside an object, or a value is expected instead of a key. + * + * \note See string(const std::string&) for the encoding rules of keys + * and string values. + */ +JSONWriter& JSONWriter::key(const std::string& key) { + assert_expects_key(); + encode_json_string(os_, key); + end_.push_back(':'); + return *this; +} + +/*! + * \brief Finishes / closes / pops the current object of the JSON document. + * + * \return this JSON writer, for call chaining. + * \throw std::runtime_exception() if the current \em scope is not + * inside an object. + */ +JSONWriter& JSONWriter::pop_object() { + if (end_.empty()) { + throw std::runtime_error("No open object to end"); + } + if (end_.back() == ',') end_.pop_back(); // discard + if (end_.empty() || end_.back() != '}') { + throw std::runtime_error("No open object in this scope"); + } + flush_pending_ws(); + os_ << end_.back(); + end_.pop_back(); + end_.push_back(','); + return *this; +} + +/*! + * \brief Writes explicit whitespace to the underlying stream. + * + * This writer does not add any whitespace when generating the text of the + * JSON document, but explicit whitespace can be added via this method. + * + * \param ws the whitespace to append to the stream. + * \param before_comma_colon whether the whitespace should come before + * a pending comma (separating values of an array, or key-value pairs + * of an object), or pending colon (separating the key and value in an + * object's key-value pair). + * \return this JSON writer, for call chaining. + * + * \note JSON recognizes only four characters as insignificant whitespace: + * Space 0x20, Horizontal Tab 0x09, New Line / Line Feed 0x0A, and + * Carriage Return 0x0D. This differs from the C/C++ isspace() method + * which adds Vertical Tab 0x0B and Form Feed 0x0C. + * + * \throw std::runtime_exception() if the current \em scope is not + * inside an object or array. + */ +JSONWriter& JSONWriter::whitespace(const std::string& ws, bool before_comma_colon) { +#if defined (OES_DEBUG_BUILD) + for (size_t i = 0; i < ws.length(); ++i) { + unsigned char c = ws[i]; + if (c > 127 || !is_json_whitespace(c)) { + throw std::runtime_error("invalid whitespace"); + } + } +#endif + + // the ok() test deals with whitespace before/after the top-level value + if (before_comma_colon || ok()) { + os_ << ws; + } else { + const char* s = ws.c_str(); + pending_ws_.insert(pending_ws_.end(), s, s + ws.length()); + } + return *this; +} + +/*! + * \internal + * \brief Flush any pending whitespace. + */ +void JSONWriter::flush_pending_ws() { + if (!pending_ws_.empty()) { + os_.write(&pending_ws_.front(), pending_ws_.size()); + pending_ws_.clear(); + } +} + +/*! + * \fn template JSONWriter& JSONWriter::value(const std::vector&) + * \brief Writes a value for which there is an available json_write() overload. + * + * This function expects a json_write(JSONWriter& json, const V&) to be + * defined in the oes::core namespace, or defined in the same namespace + * as V, to be found via argument-dependent lookup (ADL). + * + * \tparam V type of the value + * \param value the value to write. + * \return this JSON writer, for call chaining. + * + * \sa to_json_text() + */ + +/*! + * \fn template JSONWriter& JSONWriter::values(const C& container) + * \brief Writes a (non-associative) container of values. + * + * Writes \p values to this JSON writer as a JSON array. + * + * This function expects a json_write(JSONWriter& json, const V&) to + * write the vector values, where V stands for \em typename \em C::value_type. + * Overloads already exist for all JSON basic types, and additional overloads + * for custom types can be used as well (See notes in to_json_text()). + * + * \tparam C STL container of values. + * \param container a container of values + * \return this JSON writer, for call chaining. + */ + +/*! + * \fn template JSONWriter& JSONWriter::size(const C& container) + * \brief Writes the size of an STL container. + * + * Convenience for writing the size of an STL container. + * Equivalent to: \code .size(container.size()) \endcode. + * + * \tparam C STL container of values. + * \param container a container of values + * \return this JSON writer, for call chaining. + */ + +/*! + * \fn JSONWriter& JSONWriter::size(size_t value) + * \brief Writes the size of an STL container. + * + * Convenience for writing a size in a 32/64-bit portable manner. + * Equivalent to: \code .size(static_cast(value)) \endcode. + * + * \tparam C STL container of values. + * \param value the size to write as a integral number. + * \return this JSON writer, for call chaining. + */ + +/*************************************************************************/ + +/*! \class JSONHandler oes/core/utils/Json.h + * \brief Handles parsing events from a JSON document. + * + * Abstract class to be extended by a client to interpret a JSON document + * logically, independently of the low-level decoding of the text of that + * document to restore proper primitive values as numbers, string, booleans, + * and without having to check that the JSON document is well-formed (the + * parser checks well-formed-ness, and will throw an exception in that case). + * + * A JSON parser calls back a different method depending on the parsing event, + * and that methods implementation can either accept that event by returning + * true, or reject it by returning false, which aborts parsing. This allows + * for example a handler to accept only some JSON documents which conform to + * an expected structure, like for example an array of strings only. + * + * By default all handling methods return false, thus our array of strings + * handler needs only overriding handle_array_begin(), handle_array_end(), + * and handle_string() and in effect reject any document that contains + * primitives other than a string, or that contains an array. + * + * A JSONHandler is similar to a SAX handler for XML documents. + */ + +/*! + * \brief Do nothing constructor. + * + * \param return_code (Optional; defaults to false) the fixed return code for + * all handle_* methods of this handler. + */ +JSONHandler::JSONHandler(bool return_code) : return_code(return_code) { +} + +/*! + * \brief Do nothing destructor. + */ +JSONHandler::~JSONHandler() { +} + +/*! + * \brief Accept the start of an object. + * \return true if an object is allowed in this context; + * false otherwise (default), which aborts the parsing. + */ +bool JSONHandler::handle_object_begin() { + return return_code; +} + +/*! + * \brief Accept an object's key. + * \param key the object's key. Unicity of the key within the current object + * is not verified. + * \return true if an object key is allowed in this context; + * false otherwise (default), which aborts the parsing. + */ +bool JSONHandler::handle_object_key(const std::string& key) { + OES_UNUSED(key); + return return_code; +} + +/*! + * \brief Hook for handler to process the key's value the parser + * just finished dispatching to the handler. + * + * Unlike most handle_ methods, there is no equivalent method in JSONWriter + * because the command is pure JSON syntax, not a value. As such, there is + * nothing to \em handle really, but it allows the handler code to be notified + * more simply when the value associated to a key, which might be complex, + * finished being parsed/handled. + * + * \return always true. + * + * \note handle_object_comma() is not called after reading the last key's value + * so handler code depending on handle_object_comma() might call it explicitly + * from its handle_object_end() method, to \em process the last value using + * handle_object_comma() as well. + */ +bool JSONHandler::handle_object_comma() { + return true; +} + +/*! + * \brief Accept the end of an object. + * \return true if an object is complete in this context; + * false otherwise (default), which aborts the parsing. + * \note the parser checks we are in the context of an open object, and + * this handler does not need to check that. + */ +bool JSONHandler::handle_object_end() { + return return_code; +} + +/*! + * \brief Accept the start of an array. + * \return true if an object is allowed in this context; + * false otherwise (default), which aborts the parsing. + */ +bool JSONHandler::handle_array_begin() { + return return_code; +} + +/*! + * \brief Hook for handler to process the array's value the parser + * just finished dispatching to the handler. + * + * Unlike most handle_ methods, there is no equivalent method in JSONWriter + * because the command is pure JSON syntax, not a value. As such, there is + * nothing to \em handle really, but it allows the handler code to be notified + * more simply when the current value of the array, which might be complex, + * finished being parsed/handled. + * + * \return always true. + * + * \note handle_object_comma() is not called after reading the last array value + * so handler code depending on handle_object_comma() might call it explicitly + * from its handle_array_end() method, to \em process the last value using + * handle_object_comma() as well. + */ +bool JSONHandler::handle_array_comma() { + return true; +} + +/*! + * \brief Accept the end of an array. + * \return true if an array is complete in this context; + * false otherwise (default), which aborts the parsing. + * \note the parser checks we are in the context of an open array, and + * this handler does not need to check that. + */ +bool JSONHandler::handle_array_end() { + return return_code; +} + +/*! + * \brief Accept a primitive signed integer value. + * \param value the integer to handle. + * \return true if an integer is allowed in this context; + * false otherwise (default), which aborts the parsing. + * \note the parser decide which handle_number overload to call based on the + * presence or absence of a decimal-point and exponent, and the value + * itself, preferring this method over the 64-bits ones for integer if + * it fits inside a 32-bit int. + * \note A number originally written using JSONWriter::number(double), e.g. 1.0 + * can result at parse-time to a call to handle_number(int) since was + * shortened to the equivalent text "1" with no loss of precision. + */ +bool JSONHandler::handle_number(int value) { + OES_UNUSED(value); + return return_code; +} + +/*! + * \brief Accept a primitive 64-bit unsigned integer value. + * \param value the integer to handle. + * \return true if an integer is allowed in this context; + * false otherwise (default), which aborts the parsing. + * \note the parser decide which handle_number overload to call based on the + * presence or absence of a decimal-point and exponent, and the value + * itself, preferring this method over the 32-bit one for integer if + * it does not fit inside a 32-bit int. + */ +bool JSONHandler::handle_number(uint64_t value) { + OES_UNUSED(value); + return return_code; +} + +/*! + * \brief Accept a primitive 64-bit signed integer value. + * \param value the integer to handle. + * \return true if an integer is allowed in this context; + * false otherwise (default), which aborts the parsing. + + * \note the parser decide which handle_number overload to call based on the + * presence or absence of a decimal-point and exponent, and the value + * itself, preferring this method over the 32-bit one for integer if + * it does not fit inside a 32-bit int. + */ +bool JSONHandler::handle_number(int64_t value) { + OES_UNUSED(value); + return return_code; +} + +/*! + * \brief Accept a primitive floating-point value. + * \param value the floating-point to handle. + * \return true if an floating-point is allowed in this context; + * false otherwise (default), which aborts the parsing. + * \note A number originally written using JSONWriter::number(double), e.g. 1.0 + * can result at parse-time to a call to handle_number(int) instead of this + * method since was shortened to the equivalent text "1" with no loss of + * precision, and is read back as an integer. + * \warning The precision used for the std::ostream, if using a JSONWriter, + * may result in truncation which could yield a different number later on + * at parse-time. + */ +bool JSONHandler::handle_number(double value) { + OES_UNUSED(value); + return return_code; +} + +/*! + * \brief Accept a primitive string value. + * \param value the string to handle. This string is a view of an internal + * buffer of the parser that will be reused immediately after this call + * and should therefore be copied to a client-side buffer if it should + * be kept by the client. \a value is always null-terminated. + * \return true if a string is allowed in this context; + * false otherwise (default), which aborts the parsing. + * \note The string has been decoded, except for UTF-8 encoded multi-byte + * Unicode codepoints which are left unchanged. + */ +bool JSONHandler::handle_string(const std::string& value) { + OES_UNUSED(value); + return return_code; +} + +/*! + * \brief Accept a primitive boolean value. + * \param value the boolean (literal "true" or "false") to handle. + * \return true if a boolean is allowed in this context; + * false otherwise (default), which aborts the parsing. + */ +bool JSONHandler::handle_boolean(bool value) { + OES_UNUSED(value); + return return_code; +} + +/*! + * \brief Accept a primitive null value. + * \return true if a null is allowed in this context; + * false otherwise (default), which aborts the parsing. + */ +bool JSONHandler::handle_null() { + return return_code; +} + +/*! + * \brief Accept structurally insignificant whitespace. + * \return true (default) if whitespace is allowed in this context; + * false otherwise, which aborts the parsing. + */ +bool JSONHandler::handle_whitespace( + const std::string& ws, bool before_comma_colon +) { + OES_UNUSED(ws); + OES_UNUSED(before_comma_colon); + return return_code; // i.e. silently ignore +} + +/*************************************************************************/ + +namespace { + +/*! + * \brief Handler with forwards all JSON parsing events to a JSONWriter. + * + * Mostly useful for round-trip unit testing. + */ +class JSONWriterHandler : public JSONHandler { +public: + JSONWriterHandler(std::ostream& os); + JSONWriterHandler(const JSONWriter_ptr& writer); + virtual ~JSONWriterHandler(); + + virtual bool handle_object_begin(); + virtual bool handle_object_key(const std::string& key); + virtual bool handle_object_end(); + + virtual bool handle_array_begin(); + virtual bool handle_array_end(); + + virtual bool handle_number(int value); + virtual bool handle_number(int64_t value); + virtual bool handle_number(uint64_t value); + virtual bool handle_number(double value); + virtual bool handle_string(const std::string& value); + virtual bool handle_boolean(bool value); + virtual bool handle_null(); + + virtual bool handle_whitespace(const std::string& ws, bool before_comma_colon); + +private: + JSONWriter_ptr writer_ptr_; + JSONWriter& writer_; +}; + +/*! + * \brief Instantiates this handler, instantiating an internal JSONWriter + * around the given output stream. + * \param os the output stream to rewrite the JSON document to. + */ +JSONWriterHandler::JSONWriterHandler( + std::ostream& os +) : writer_ptr_(new JSONWriter(os)) + , writer_(*writer_ptr_) +{ +} + +/*! + * \brief Instantiates this handler, around an existing JSONWriter. + * \param writer the JSON writer to forward all parsing events to. + */ +JSONWriterHandler::JSONWriterHandler( + const JSONWriter_ptr& writer +) : writer_ptr_(writer) + , writer_(*writer_ptr_) +{ + BOOST_VERIFY(writer_ptr_); +} + +/*! + * \brief Deletes this handler. + * Resets the internal JSONWriter + * \sa JSONWriter::reset() + */ +JSONWriterHandler::~JSONWriterHandler() { + writer_.reset(); +} + +/*! + * \brief Handles a new object. + * \return Always true. + * \note calls JSONWriter::push_object() + */ +bool JSONWriterHandler::handle_object_begin() { + writer_.push_object(); + return true; +} + +/*! + * \brief Handles an object key of a key-value pair. + * \param key an object key name. + * \return Always true. + * \note calls JSONWriter::key(key) + */ +bool JSONWriterHandler::handle_object_key(const std::string& key) { + writer_.key(key); + return true; +} + +/*! + * \brief Handles the end of an object. + * \return Always true. + * \note calls JSONWriter::pop_object() + */ +bool JSONWriterHandler::handle_object_end() { + writer_.pop_object(); + return true; +} + +/*! + * \brief Handles a new array. + * \return Always true. + * \note calls JSONWriter::push_array() + */ +bool JSONWriterHandler::handle_array_begin() { + writer_.push_array(); + return true; +} + +/*! + * \brief Handles the end of an array. + * \return Always true. + * \note calls JSONWriter::pop_array() + */ +bool JSONWriterHandler::handle_array_end() { + writer_.pop_array(); + return true; +} + +/*! + * \brief Handles a primitive integer number value. + * \param value an integer value. + * \return Always true. + * \note calls JSONWriter::number(value) + */ +bool JSONWriterHandler::handle_number(int value) { + writer_.number(value); + return true; +} + +/*! + * \brief Handles a primitive integer number value. + * \param value an integer value. + * \return Always true. + * \note calls JSONWriter::number(value) + */ +bool JSONWriterHandler::handle_number(int64_t value) { + writer_.number(value); + return true; +} + +/*! + * \brief Handles a primitive unsigned integer number value. + * \param value an integer value. + * \return Always true. + * \note calls JSONWriter::number(value) + */ +bool JSONWriterHandler::handle_number(uint64_t value) { + writer_.number(value); + return true; +} + +/*! + * \brief Handles a primitive floating-point number value. + * \param value an floating-point value. + * \return Always true. + * \note calls JSONWriter::number(value) + */ +bool JSONWriterHandler::handle_number(double value) { + writer_.number(value); + return true; +} + +/*! + * \brief Handles a primitive string value. + * \param value a string value. + * \return Always true. + * \note calls JSONWriter::string(value) + */ +bool JSONWriterHandler::handle_string(const std::string& value) { + writer_.string(value); + return true; +} + +/*! + * \brief Handles a primitive boolean literal value. + * \param value a boolean true or false value. + * \return Always true. + * \note calls JSONWriter::boolean(value) + */ +bool JSONWriterHandler::handle_boolean(bool value) { + writer_.json_bool(value); + return true; +} + +/*! + * \brief Handles a primitive null literal value. + * \return Always true. + * \note calls JSONWriter::json_null() + */ +bool JSONWriterHandler::handle_null() { + writer_.json_null(); + return true; +} + +/*! + * \brief Handles insignificant whitespace. + * \param ws the JSON whitespace. + * \param before_comma_colon whether the whitespace was seen before or after + * a comma (separating array values or object key-value pairs) or a colon + * (separating an object key and value from one of its pairs). + * \a before_comma_colon is always false if either end of \a ws does not + * \em touch a comma or colon (for example in an empty array "[ ]") + * \return Always true. + * \note calls JSONWriter::whitespace(ws, before_comma_colon) + */ +bool JSONWriterHandler::handle_whitespace( + const std::string& ws, bool before_comma_colon +) { + writer_.whitespace(ws, before_comma_colon); + return true; +} + +} // namespace + +/*************************************************************************/ + +JSONHandler_ptr json_handler_writer(std::ostream& os) { + return JSONHandler_ptr(new JSONWriterHandler(os)); +} + +JSONHandler_ptr json_handler_writer(const JSONWriter_ptr& writer) { + return JSONHandler_ptr(new JSONWriterHandler(writer)); +} + +/*************************************************************************/ + +JSONToken::JSONToken( + const char* begin, const char* end +) : beg_(begin) + , end_(end? end: begin) +{ +} + +JSONRawStringRef::JSONRawStringRef( + const char* begin, const char* end +) : JSONToken(begin, end) +{ + num_.is_negative_ = num_.has_dot_ = num_.has_exp_ = false; +} + +bool JSONToken::empty() const { + BOOST_ASSERT(end_ >= beg_); + return end_ == beg_; +} + +size_t JSONToken::size() const { + BOOST_ASSERT(end_ >= beg_); + return end_ - beg_; +} + +bool JSONToken::is_quoted() const { + return size() >= 2 && *beg_ == '"' && *(end_ - 1) == '"'; +} + +const char* JSONToken::find(char c) const { + for (const char* s = beg_; s < end_; ++s) { + if (*s == c) { + return s; + } + } + return end_; +} + +bool JSONToken::matches(const char* text, size_t len) const { + return size() == len && 0 == strncmp(beg_, text, len); +} + +bool JSONToken::matches_unquote(const char* text, size_t len) const { + return size() == (len + 2) && 0 == strncmp(beg_ + 1, text, len); +} + +JSONRawStringRef& JSONRawStringRef::reset(const char* begin) { + beg_ = end_ = begin; + num_.is_negative_ = num_.has_dot_ = num_.has_exp_ = false; + return *this; +} + +JSONRawStringRef& JSONRawStringRef::endsAt(const char* end) { + end_ = end; + return *this; +} + +void JSONRawStringRef::decode(std::string& buffer, bool unquote) const { + const char* s = unquote? (beg_ + 1): beg_; + const char*const end = unquote? (end_ - 1): end_; + + // in the simplest case, no need to rescan the text. + if (!str_.has_escape_ && !str_.has_unicode_) { + buffer.append(s, end); + return; + } + + buffer.reserve(end - s); + for (char c = *s; s < end; c = *++s) { + if (c != '\\') { + buffer.push_back(c); + continue; + } + + BOOST_ASSERT((s + 1) < end); // already checked by parser + + switch (c = *++s) { + case 'u': { + // encoded Unicode codepoint must be followed by 4 hex + BOOST_ASSERT((s + 4) < end); // already checked by parser + + ++s; + imembuf rd(s, s + 4); + s += 3; // not 4, since loop itself will also ++ + std::istream is(&rd); + unsigned short codepoint = 0; + is >> std::hex >> codepoint; + BOOST_ASSERT(!is.fail()); // impossible, 4 xdigit must work + + // decode codepoint + if (codepoint < 0x0080) { // characters 0x0000 - 0x007F + buffer.push_back(0x00 | (char)((codepoint & 0x007F) >> 0)); + } else if (codepoint < 0x0800) { // characters 0x0080 - 0x07FF + buffer.push_back(0xC0 | (char)((codepoint & 0x07C0) >> 6)); + buffer.push_back(0x80 | (char)((codepoint & 0x003F) >> 0)); + } else if (0xD800 <= codepoint && codepoint <= 0xDFFF) { + // surrogate codepoints + // TODO: handle surrogate pairs > BMP! + throw std::runtime_error("TODO: Support Unicode Surrogate Pairs"); + } else { // characters 0x0800 - 0xD7FF and 0xE000 - 0xFFFF + buffer.push_back(0xE0 | (char)((codepoint & 0xF000) >> 12)); + buffer.push_back(0x80 | (char)((codepoint & 0x0FC0) >> 6)); + buffer.push_back(0x80 | (char)((codepoint & 0x003F) >> 0)); + } + break; + } + + case '"': case '\\': case '/': + buffer.push_back(c); + break; + + case 'b': buffer.push_back('\b'); break; + case 'f': buffer.push_back('\f'); break; + case 'n': buffer.push_back('\n'); break; + case 'r': buffer.push_back('\r'); break; + case 't': buffer.push_back('\t'); break; + + default: // only above characters are allowed after backslash + throw std::runtime_error("Invalid String: Unexpected Escape"); + } + } +} + +std::string JSONToken::to_string(bool unquote) const { + BOOST_ASSERT(!unquote || size() >= 2); + return unquote? std::string(beg_ + 1, end_ - 1): std::string(beg_, end_); +} + +std::ostream& JSONToken::print(std::ostream& os) const { + return os.write(beg_, size()); +} + +/*************************************************************************/ + +namespace { + +void do_nothing_handler(JSONParsingEventType, const JSONRawStringRef&, void*) { + return; +} + +void print_no_whitespace_handler( + JSONParsingEventType evt, const JSONRawStringRef& text, void* user_data +) { + BOOST_ASSERT(!text.empty()); + if (evt != JSON_SYNTAX_WHITESPACE) { + std::ostream* os = static_cast(user_data); + os->write(text.beg_, text.size()); + //std::string value = text.to_string(); + //(*os) << value; + } +} + +void type_detection_handler( + JSONParsingEventType evt, const JSONRawStringRef& /*text*/, void* user_data +) { + JSONType& type = *static_cast(user_data); + if (type != JSON_TYPE_INVALID) { + // Type is already known + return; + } + + switch (evt) { + case JSON_OBJECT_BEGIN: + type = JSON_TYPE_OBJECT; + break; + case JSON_ARRAY_BEGIN: + type = JSON_TYPE_ARRAY; + break; + case JSON_STRING: + type = JSON_TYPE_STRING; + break; + case JSON_NUMBER: + type = JSON_TYPE_NUMBER; + break; + case JSON_TRUE: + case JSON_FALSE: + type = JSON_TYPE_BOOLEAN; + break; + case JSON_NULL: + type = JSON_TYPE_OBJECT; + break; + default: + break; + } +} + +/*! + * \internal + * \brief SAX-like hand-written parser for JSON documents. + */ +class JSONSyntaxParser { // TODO: Rename JSONPushParser + JSONSyntaxHandler handler_; + void* user_data_; + JSONRawStringRef text_; + +public: + explicit JSONSyntaxParser(JSONSyntaxHandler handler, void* user_data = 0); + ~JSONSyntaxParser(); + + // TODO: Pull text from some "input stream" + // TOOD: Allow parsing suspend()/resume() for incomplete "input streams" + size_t parse(const char* json_utf8_text, size_t len); + +private: + JSONSyntaxParser(const JSONSyntaxParser&); + JSONSyntaxParser& operator=(const JSONSyntaxParser&); + + // WARNING: JSONPullParser copy/paste's (with slight modifications) these + // methods, so changes here must be reflected there, and vice-versa. + void consume_char(const char*& s, const char*const end, bool at_end_ok = false); + bool consume_whitespace(const char*& s, const char*const end); + bool expect_unicode(const char*& s, const char*const end); + void expect_string(const char*& s, const char*const end, bool is_key = false); + void expect_literal(const char*& s, const char*const end, const char* lit, int lit_len); + void expect_number(const char*& s, const char*const end); + void expect_array_content(const char*& s, const char*const end); + void expect_object_content(const char*& s, const char*const end); + void expect_object_element(const char*& s, const char*const end); + void expect_value(const char*& s, const char*const end); +}; + +JSONSyntaxParser::JSONSyntaxParser( + JSONSyntaxHandler handler, void* user_data +) : handler_(handler) + , user_data_(user_data) +{ +} + +JSONSyntaxParser::~JSONSyntaxParser() +{ +} + +void JSONSyntaxParser::consume_char(const char*& s, const char*const end, bool at_end_ok) { + if (++s >= end) { + if (!(at_end_ok && s == end)) { + throw std::runtime_error("Unexpected end-of-value"); + } + } +} + +bool JSONSyntaxParser::consume_whitespace(const char*& s, const char*const end) { + text_.reset(s); + const char*const prev_s = s; + for (; s < end; ++s) { + unsigned char c = *s; + if (c > 127 || !is_json_whitespace(c)) { + if (s > text_.beg_) { + text_.before_comma_colon_ = (*s == ',') || (*s == ':'); + handler_(JSON_SYNTAX_WHITESPACE, text_.endsAt(s), user_data_); + } + return false; + } + } + if (end > prev_s) { + // text_.before_comma_colon_ implicitly false from reset() + handler_(JSON_SYNTAX_WHITESPACE, text_.endsAt(end), user_data_); + } + return true; +} + +// only verifies the codepoint, does not extract its value +bool JSONSyntaxParser::expect_unicode(const char*& s, const char*const end) { + BOOST_ASSERT(*s == 'u'); + if ((s + 4) >= end) { + throw std::runtime_error("Truncated Unicode Codepoint"); + } + if (!isxdigit(*++s) || !isxdigit(*++s) || + !isxdigit(*++s) || !isxdigit(*++s) + ) { + throw std::runtime_error("Invalid Unicode Codepoint"); + } + return text_.str_.has_unicode_ = true; +} + +// positions \a s one-char past the closing double-quote +void JSONSyntaxParser::expect_string( + const char*& s, const char*const end, bool is_key +) { + BOOST_ASSERT(*s == '"'); + text_.reset(s); + consume_char(s, end); + //const char* start = s; + for (unsigned char c = *s; s < end; c = *++s) { + if (c > 127) { + // part of a multi-byte UTF-8 encoded string. + // TODO: Check valid multi-byte. + text_.str_.not_ascii_ = true; + continue; + } + switch (c) { + case '"': { + // Reached the end of the string. + consume_char(s, end, true); + handler_( + is_key? JSON_OBJECT_KEY: JSON_STRING, + text_.endsAt(s), user_data_ + ); + return; + } + case '\\': + if ((s + 1) >= end) { + throw std::runtime_error("Unexpected Partial Escape"); + } + switch (c = *++s) { + case 'u': // encoded Unicode codepoint must be followed by 4 hex + JSONSyntaxParser::expect_unicode(s, end); + break; + + case '"': case '\\': case '/': + text_.str_.has_escape_ = true; + break; + + case 'b': case 'f': case 'n': case 'r': case 't': + text_.str_.has_escape_ = true; + break; + + default: // only above characters are allowed after backslash + throw std::runtime_error("Invalid String: Unexpected Escape"); + } + break; + + case '/': + // Even though forward-slash can be escaped, as per the spec, + // (and JSONWriter always escape it), it is also valid to have an + // unescaped one in an encoded JSON string. See Errata ID: 3159 + // from http://www.rfc-editor.org/errata_search.php?rfc=4627 + //throw std::runtime_error("Invalid String: Unescaped forward-slash"); + break; + + default: + if (!isprint(c)) { + // should have been encoded in range \u0000-\u001f + throw std::runtime_error("Invalid String: Unencoded non-printable character"); + } + } + } + // reached the end without seeing the closing double-quote + throw std::runtime_error("Invalid String: No closing double-quote"); +} + +void JSONSyntaxParser::expect_literal( + const char*& s, const char*const end, + const char* lit, int lit_len +) { + if ((s + lit_len) > end) { + throw std::runtime_error("Unexpected end-of-value"); + } + if (0 != strncmp(s, lit, lit_len)) { + throw std::runtime_error("Invalid Value"); + } + text_.reset(s); + text_.endsAt(s += lit_len); +} + +void JSONSyntaxParser::expect_number(const char*& s, const char*const end) { + BOOST_ASSERT(*s == '-' || isdigit(*s)); + + text_.reset(s); + + if (*s == '-') { // optional + text_.num_.is_negative_ = true; + consume_char(s, end); + } + + // required part + if (*s == '0') { + consume_char(s, end, true); + if (s >= end) { + // just 0 is possible in naked 0 integer value. + handler_(JSON_NUMBER, text_.endsAt(s + 1), user_data_); + return; + } + // a leading 0 must be followed by a decimal-point, as leading + // zeros are not allowed in encoded JSON numbers. + if (*s != '.' && isdigit(*s)) { + throw std::runtime_error("Invalid number: Expected decimal-point"); + } + } else { + // expect_size_t + if (!isdigit(*s)) { + throw std::runtime_error("Invalid number: Expected digits"); + } + for (unsigned char c = *s; s < end; c = *++s) { + if (c < '0' || '9' < c) { + break; + } + } + } + + // optional fractional part (after the dot) + if (s < end && *s == '.') { + text_.num_.has_dot_ = true; + consume_char(s, end); // must be followed by digits + // consume the digits that follow + const unsigned char first_c = *s; + if (first_c < '0' || '9' < first_c) { + throw std::runtime_error("Invalid number: expected decimals"); + } + consume_char(s, end, true); // may have more digits + for (unsigned char c = *s; s < end; c = *++s) { + if (c < '0' || '9' < c) { + break; + } + } + } + + // optional exponential part + if (s < end && (*s == 'e' || *s == 'E')) { + text_.num_.has_exp_ = true; + consume_char(s, end); // must be followed by +/- and/or digits + if (*s == '-' || *s == '+') { + consume_char(s, end); // must be followed by digits + } + const unsigned char first_c = *s; + if (first_c < '0' || '9' < first_c) { + throw std::runtime_error("Invalid number: expected decimals"); + } + consume_char(s, end, true); // may have more digits + for (unsigned char c = *s; s < end; c = *++s) { + if (c < '0' || '9' < c) { + break; + } + } + } + + handler_(JSON_NUMBER, text_.endsAt(s), user_data_); +} + +void JSONSyntaxParser::expect_array_content(const char*& s, const char*const end) { + BOOST_ASSERT(*s == '['); + handler_(JSON_ARRAY_BEGIN, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(s, end); + + bool expect_comma = false; + while (s < end) { + consume_whitespace(s, end); + if (*s == ']') { + handler_(JSON_ARRAY_END, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(s, end, true); + return; + } + + if (expect_comma) { + if (*s == ',') { + handler_(JSON_SYNTAX_ARRAY_COMMA, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(s, end); + } else { + throw std::runtime_error("Invalid array: missing comma separator"); + } + } + + expect_value(s, end); + expect_comma = true; + } + throw std::runtime_error("Unexpected end-of-array"); +} + +void JSONSyntaxParser::expect_object_element(const char*& s, const char*const end) { + if (*s != '"') { + throw std::runtime_error("Missing expected object key"); + } + expect_string(s, end, true); + consume_whitespace(s, end); + + if (*s != ':') { + throw std::runtime_error("Missing expected object key:value separator"); + } + text_.reset(s); + consume_char(s, end); + handler_(JSON_SYNTAX_OBJECT_COLON, text_.endsAt(s), user_data_); + expect_value(s, end); +} + +void JSONSyntaxParser::expect_object_content(const char*& s, const char*const end) { + BOOST_ASSERT(*s == '{'); + handler_(JSON_OBJECT_BEGIN, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(s, end); + + bool expect_comma = false; + while (s < end) { + consume_whitespace(s, end); + if (*s == '}') { + handler_(JSON_OBJECT_END, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(s, end, true); + return; + } + + if (expect_comma) { + if (*s == ',') { + handler_(JSON_SYNTAX_OBJECT_COMMA, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(s, end); + consume_whitespace(s, end); + } else { + throw std::runtime_error("Invalid object: missing comma separator"); + } + } + + expect_object_element(s, end); + expect_comma = true; + } + throw std::runtime_error("Unexpected end-of-object"); +} + +void JSONSyntaxParser::expect_value(const char*& s, const char*const end) { + if (consume_whitespace(s, end)) { + throw std::runtime_error("Unexpected end-of-value"); + } + + switch (*s) { + case '{': + expect_object_content(s, end); + return; + + case '[': + expect_array_content(s, end); + return; + + case '"': + expect_string(s, end); + return; + + case '-': // case '+': leading plus not allowed by JSON spec + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + expect_number(s, end); + return; + + case 't': + expect_literal(s, end, "true", 4); + handler_(JSON_TRUE, text_, user_data_); + return; + case 'f': + expect_literal(s, end, "false", 5); + handler_(JSON_FALSE, text_, user_data_); + return; + case 'n': + expect_literal(s, end, "null", 4); + handler_(JSON_NULL, text_, user_data_); + return; + } + + throw std::runtime_error("Invalid JSON value: Unexpected character"); +} + +size_t JSONSyntaxParser::parse(const char* json_utf8_text, size_t len) { + const char* s = json_utf8_text; + const char*const end = json_utf8_text + len; + expect_value(s, end); + consume_whitespace(s, end); // ignore trailing whitespace + const ptrdiff_t remaining = end - s; + return (remaining <= 0)? 0: remaining; // zero if we consumed all input +} + +/*************************************************************************/ + +struct ValueHandler { + JSONHandler& handler_; + std::string buffer_; + + ValueHandler( + JSONHandler& handler + ) : handler_(handler) + { + buffer_.reserve(15); // small string opt limit? + } + + bool handle_whitespace(const JSONRawStringRef& text) { + buffer_.assign(text.beg_, text.end_); + return handler_.handle_whitespace(buffer_, text.before_comma_colon_); + } + + bool handle_string(const JSONRawStringRef& text) { + buffer_.clear(); + text.decode(buffer_, true); + return handler_.handle_string(buffer_); + } + + bool handle_object_key(const JSONRawStringRef& text) { + buffer_.clear(); + text.decode(buffer_, true); + return handler_.handle_object_key(buffer_); + } + + bool handle_number(const JSONRawStringRef& text) { + // Get the int part. + const bool is_negative = text.num_.is_negative_; + const char* beg = is_negative? text.beg_ + 1: text.beg_; + const char* end = text.end_; + if (text.num_.has_dot_) { + end = text.find('.'); + BOOST_ASSERT(end != text.end_); + } else if (text.num_.has_exp_) { + end = text.find('e'); + if (end == text.end_) { + end = text.find('E'); + } + BOOST_ASSERT(end != text.end_); + } + + const uint64_t int_part = parser_integer(beg, end); + + if (!text.num_.has_dot_ && !text.num_.has_exp_) { + // naked integer value + if (int_part <= (uint64_t)std::numeric_limits::max()) { + const int as_int32 = static_cast(int_part); + return handler_.handle_number(is_negative? -as_int32: as_int32); + + } else if (is_negative && int_part == (1 + (uint64_t)std::numeric_limits::max())) { + // smallest int, i.e. -2,147,483,648 does not fit in absolute value + // as an int, so we cannot cast int_part into as_int32 as above. + return handler_.handle_number(std::numeric_limits::min()); + + } else if (int_part <= (uint64_t)std::numeric_limits::max()) { + const int64_t as_int64 = static_cast(int_part); + return handler_.handle_number(is_negative? -as_int64: as_int64); + + } else if (is_negative && int_part == (1 + (uint64_t)std::numeric_limits::max())) { + // smallest int64, i.e. -9,223,372,036,854,775,808 does not fit in absolute value + // as an int64, so we cannot cast int_part into as_int64 as above. + return handler_.handle_number(std::numeric_limits::min()); + + } else if (!is_negative && int_part <= std::numeric_limits::max()) { + return handler_.handle_number(int_part); + + } else { + if (is_negative) { + BOOST_ASSERT(int_part > (uint64_t)std::numeric_limits::max() + 1); + throw std::runtime_error("Numeric overflow: < MIN_SSIZE_T"); + } else { + // we already throw in parse_number(), so should never be reached + throw std::runtime_error("Numeric overflow: > MAX_SIZE_T"); + } + } + } + + // not an integer, so parse as a double + const double value = parse_number(text.beg_, text.end_); + return handler_.handle_number(value); + } + + static void check(bool rc) { + if (!rc) { + throw std::runtime_error("Handler Stop"); + } + } + + static void handle_event( + JSONParsingEventType evt, const JSONRawStringRef& text, void* user_data + ) { + BOOST_ASSERT(!text.empty()); + ValueHandler& self = *static_cast(user_data); + switch (evt) { + case JSON_NULL: check(self.handler_.handle_null()); break; + case JSON_TRUE: check(self.handler_.handle_boolean(true)); break; + case JSON_FALSE: check(self.handler_.handle_boolean(false)); break; + + case JSON_STRING: check(self.handle_string(text)); break; + case JSON_NUMBER: check(self.handle_number(text)); break; + + case JSON_OBJECT_BEGIN: check(self.handler_.handle_object_begin()); break; + case JSON_OBJECT_KEY: check(self.handle_object_key(text)); break; + case JSON_OBJECT_END: check(self.handler_.handle_object_end()); break; + + case JSON_ARRAY_BEGIN: check(self.handler_.handle_array_begin()); break; + case JSON_ARRAY_END: check(self.handler_.handle_array_end()); break; + + // syntax-only events + case JSON_SYNTAX_WHITESPACE: check(self.handle_whitespace(text)); break; + case JSON_SYNTAX_OBJECT_COLON: break; + case JSON_SYNTAX_OBJECT_COMMA: check(self.handler_.handle_object_comma()); break; + case JSON_SYNTAX_ARRAY_COMMA: check(self.handler_.handle_array_comma()); break; + + default: + throw std::runtime_error("Unexpected JSON parsing event"); + } + } + +private: + ValueHandler& operator=(const ValueHandler&); +}; + +} // namespace + +/*************************************************************************/ + +size_t json_parse_syntax( + const char* json_utf8_text, size_t len, + JSONSyntaxHandler handler, void* handler_data +) { + JSONSyntaxParser parser(handler, handler_data); + return parser.parse(json_utf8_text, len); +} + +/*************************************************************************/ + +/*! \enum JSONType oes/core/utils/Json.h + * \brief The possible types of a JSON values. + * + * Valid JSON values are: + * - JSON_TYPE_OBJECT; + * - JSON_TYPE_ARRAY; + * - JSON_TYPE_STRING; + * - JSON_TYPE_NUMBER; + * - JSON_TYPE_BOOLEAN. + * + * In addition, this enum also declares: + * - JSON_TYPE_INVALID as the \em null (or \em invalid) value type; + * - JSON_TYPE_ANY as the \em wildcard value type. + * + * \sa json_type_of() + * \sa json_type_string_of() + * \sa json_type_value_of() + */ + +/****************************************************************************/ + +/*! + * \brief Determines the type of a JSON expression + * + * Parses JSON expression \p value and returns its JSON type. Note that "null" is + * considered as a separate JSON type in the JSON literature. Here it is + * considered as an empty constant: if the expression is the string "null", + * the type object is returned. + * + * \param[in] value a string containing a JSON expression + * \return the JSON type of \p value if the expression is valid, or + * JSON_TYPE_INVALID otherwise. + * + * \see JSONType + */ +JSONType json_type_of(const std::string& value) { + try { + JSONType type = JSON_TYPE_INVALID; + JSONSyntaxParser parser(type_detection_handler, &type); + if (0 == parser.parse(value.c_str(), value.length())) { + return type; + } + } catch (const std::runtime_error&) { + // Do nothing + } + return JSON_TYPE_INVALID; +} + +/*! + * \brief Returns the string form of a JSON type + * + * \param[in] type_value a JSON type + * \return the string form of \p type_value if the type is valid, or the empty + * string otherwise + */ +std::string json_type_string_of(JSONType type_value) { + return (size_t)type_value < json_type_count ? + json_type_strings[type_value] : + json_type_strings[JSON_TYPE_INVALID]; +} + +/*! + * \brief Returns the value of a JSON type string + * + * \param[in] type_string a JSON type name + * \return the value of \p type_string as defined in JSONType + */ +JSONType json_type_value_of(const std::string& type_string) { + for (size_t type_value = 0; type_value < json_type_count; ++type_value) { + if (type_string == json_type_strings[type_value]) { + return static_cast(type_value); + } + } + return JSON_TYPE_INVALID; +} + +/*************************************************************************/ + +/*! + * \brief Parses a JSON document and forwards parsing events to a handler. + * + * Instantiates an internal JSON parser to perform low-level tokenizing and + * parsing of the base JSON grammar, and forwards logical parsing events to + * a client-supplied \a handler to interpret these events into custom C++ data + * structures. The parser performs well-formed-ness validation of the JSON + * document, to make sure the opening and closing of arrays and objects are + * correct, or the encoding of strings (and object keys) conforms to the JSON + * specification (see http://tools.ietf.org/html/rfc4627), etc... + * + * The \a handler could assemble a generic document in a manner similar to + * an XML Document Object Model (DOM) tree, or instantiate custom ad-hoc + * C++ data structures if it accepts only JSON documents of a specific + * structure; for example to instantiate a std::vector from + * a JSON array of strings, accepting "[]" (an empty array) and "["foo"]" + * and "["foo", "bar"]", but refusing "{}" (an empty object), or "[1]" + * (an array of one number), etc... + * + * Note that this parser accepts a single primitive value, outside of + * an array or object, like "1" (a single number), or "true", or ""foo"" + * (a single string). + * + * \param json_utf8_text UTF-8 encoded text of the JSON document. + * \param len the byte-length of the text in \a json_utf8_text. + * \param handler Client-provided handler to receive all logical parsing + * events the internal parser will infer from the JSON text document. + * \return true if \a json_utf8_text was entirely consumed by this parser, + * with leading and trailing JSON whitespace ignored; false if unparsed + * text remains. + * \throw std::runtime_error if the JSON document is not well-formed ( + * truncated, mismatched open/close for array/object, wrongly encoded + * string, invalid number like NaN, Infinity, etc...) or if \a handler + * returned false to any of the parsing events sent to it. + */ +bool json_parse(const char* json_utf8_text, size_t len, JSONHandler& handler) { + ValueHandler state(handler); + JSONSyntaxParser parser(&ValueHandler::handle_event, &state); + return 0 == parser.parse(json_utf8_text, len); +} + +/*************************************************************************/ + +namespace { + +struct KVMapHandler { + enum State { KV_EXPECT_NONE, KV_EXPECT_OBJ, KV_EXPECT_KEY, KV_EXPECT_VAL }; + + std::map& map_; + JSONType value_type_; + State state_; + std::string key_; + std::string value_; + bool duplicates_ok_; + + KVMapHandler( + std::map& map, + bool duplicates_ok, JSONType value_type + ) : map_(map) + , value_type_(value_type) + , state_(KV_EXPECT_OBJ) + , duplicates_ok_(duplicates_ok) + { + } + + void assert_state_transition(State expected_state, State next_state) { + if (state_ != expected_state) { + throw std::runtime_error("Unexpected JSON content"); + } + state_ = next_state; + } + + void assert_value_type(JSONType value_type) { + if (value_type_ != JSON_TYPE_ANY && value_type_ != value_type) { + throw std::runtime_error("Unexpected JSON content"); + } + } + + void check(bool inserted) { + if (!inserted && !duplicates_ok_) { + throw std::runtime_error("Duplicate JSON key: " + key_); + } + } + + void operator()(JSONParsingEventType evt, const JSONRawStringRef& text) { + switch (evt) { + case JSON_SYNTAX_WHITESPACE: + case JSON_SYNTAX_OBJECT_COLON: + case JSON_SYNTAX_OBJECT_COMMA: + return; + + case JSON_OBJECT_BEGIN: + assert_state_transition(KV_EXPECT_OBJ, KV_EXPECT_KEY); + return; + case JSON_OBJECT_KEY: + assert_state_transition(KV_EXPECT_KEY, KV_EXPECT_VAL); + key_.clear(); + text.decode(key_, true); + return; + case JSON_OBJECT_END: + assert_state_transition(KV_EXPECT_KEY, KV_EXPECT_NONE); + return; + + case JSON_TRUE: + case JSON_FALSE: + assert_state_transition(KV_EXPECT_VAL, KV_EXPECT_KEY); + assert_value_type(JSON_TYPE_BOOLEAN); + check(map_.insert(std::make_pair(key_, text.to_string())).second); + return; + case JSON_NUMBER: + assert_state_transition(KV_EXPECT_VAL, KV_EXPECT_KEY); + assert_value_type(JSON_TYPE_NUMBER); + check(map_.insert(std::make_pair(key_, text.to_string())).second); + return; + case JSON_STRING: + assert_state_transition(KV_EXPECT_VAL, KV_EXPECT_KEY); + assert_value_type(JSON_TYPE_STRING); + value_.clear(); + text.decode(value_, true); + check(map_.insert(std::make_pair(key_, value_)).second); + return; + + default: // array events, *and* null + throw std::runtime_error("Unexpected JSON content"); + } + } + + static void handle_event( + JSONParsingEventType evt, const JSONRawStringRef& text, void* user_data + ) { + BOOST_ASSERT(!text.empty()); + KVMapHandler& self = *static_cast(user_data); + self(evt, text); + } + +private: + KVMapHandler& operator=(const KVMapHandler&); +}; + +struct VectorHandler { + enum State { EXPECT_NONE, EXPECT_ARRAY, EXPECT_VAL }; + + std::vector& vector_; + JSONType value_type_; + State state_; + std::string value_; + + VectorHandler(std::vector& vector, JSONType value_type) : + vector_(vector), + value_type_(value_type), + state_(EXPECT_ARRAY) + { + } + + void assert_state_transition(State expected_state, State next_state) { + if (state_ != expected_state) { + throw std::runtime_error("Unexpected JSON content"); + } + state_ = next_state; + } + + void assert_value_type(JSONType value_type) { + if (value_type_ != JSON_TYPE_ANY && value_type_ != value_type) { + throw std::runtime_error("Unexpected JSON content"); + } + } + + void operator()(JSONParsingEventType evt, const JSONRawStringRef& text) { + switch (evt) { + case JSON_SYNTAX_WHITESPACE: + case JSON_SYNTAX_ARRAY_COMMA: + return; + + case JSON_ARRAY_BEGIN: + assert_state_transition(EXPECT_ARRAY, EXPECT_VAL); + return; + case JSON_ARRAY_END: + assert_state_transition(EXPECT_VAL, EXPECT_NONE); + return; + + case JSON_TRUE: + case JSON_FALSE: + assert_state_transition(EXPECT_VAL, EXPECT_VAL); + assert_value_type(JSON_TYPE_BOOLEAN); + vector_.push_back(text.to_string()); + return; + case JSON_NUMBER: + assert_state_transition(EXPECT_VAL, EXPECT_VAL); + assert_value_type(JSON_TYPE_NUMBER); + vector_.push_back(text.to_string()); + return; + case JSON_STRING: + assert_state_transition(EXPECT_VAL, EXPECT_VAL); + assert_value_type(JSON_TYPE_STRING); + value_.clear(); + text.decode(value_, true); + vector_.push_back(value_); + return; + + default: // object events, *and* null + throw std::runtime_error("Unexpected JSON content"); + } + } + + static void handle_event( + JSONParsingEventType evt, const JSONRawStringRef& text, void* user_data + ) { + BOOST_ASSERT(!text.empty()); + VectorHandler& self = *static_cast(user_data); + self(evt, text); + } + +private: + VectorHandler& operator=(const VectorHandler&); +}; + +} // namespace + +/*! + * \brief Parses a JSON document expected to be a list of key/values. + * + * \param json_utf8_text UTF-8 encoded text of the JSON document. + * \param len the byte-length of the text in \a json_utf8_text. + * \param key_values the parsed key/values. + * \param throw_on_duplicates (Optional; defaults to false) whether a + * duplicate key triggers an exception or not. When false, the first + * key's value prevails over subsequent values for the same key. + * \param value_type allowed value types, which can be JSON_TYPE_ANY, + * JSON_TYPE_STRING, JSON_TYPE_NUMBER or JSON_TYPE_BOOLEAN. If value_type is + * JSON_TYPE_ANY (the default), the vector accepts values of string, + * number and boolean JSON types. Otherwise, values not of the specified + * type cause the parsing to fail. + * \return true if \a json_utf8_text was entirely consumed by this parser, + * with leading and trailing JSON whitespace ignored; false if unparsed + * text remains. + * + * \throw std::runtime_error if the JSON document is not well-formed ( + * truncated, mismatched open/close for array/object, wrongly encoded + * string, invalid number like NaN, Infinity, etc...), or contains a + * duplicate key and \a throw_on_duplicates is true. + * + * \see json_write(JSONWriter&, const std::map&) + */ +bool json_parse( + const char* json_utf8_text, size_t len, + std::map& key_values, + bool throw_on_duplicates, JSONType value_type +) { + KVMapHandler state(key_values, !throw_on_duplicates, value_type); + JSONSyntaxParser parser(&KVMapHandler::handle_event, &state); + return 0 == parser.parse(json_utf8_text, len); +} + +/*! + * \brief Parses a JSON document expected to be a vector of values. + * + * \param json_utf8_text UTF-8 encoded text of the JSON document. + * \param len the byte-length of the text in \a json_utf8_text. + * \param values the parsed vector of values. + * \param value_type allowed value types, which can be JSON_TYPE_ANY, + * JSON_TYPE_STRING, JSON_TYPE_NUMBER or JSON_TYPE_BOOLEAN. If value_type is + * JSON_TYPE_ANY (the default), the vector accepts values of string, + * number and boolean JSON types. Otherwise, values not of the specified + * type cause the parsing to fail. + * \return true if \a json_utf8_text was entirely consumed by this parser, + * with leading and trailing JSON whitespace ignored; false if unparsed + * text remains. + * + * \throw std::runtime_error if the JSON document is not well-formed ( + * truncated, mismatched open/close for array/object, wrongly encoded + * string, invalid number like NaN, Infinity, etc...), or contains a + * duplicate key and \a throw_on_duplicates is true. + * + * \see json_write(JSONWriter&, const std::vector&) + */ +bool json_parse( + const char* json_utf8_text, size_t len, + std::vector& values, JSONType value_type +) { + VectorHandler state(values, value_type); + JSONSyntaxParser parser(&VectorHandler::handle_event, &state); + return 0 == parser.parse(json_utf8_text, len); +} + +/*************************************************************************/ + +/*! \enum JSONParsingEventType oes/core/utils/Json.h + * \brief The possible types of a JSON parsing event (token). + * + * Primitive JSON value tokens: + * - JSON_NULL; + * - JSON_TRUE; + * - JSON_FALSE; + * - JSON_NUMBER; + * - JSON_STRING. + * + * Structured JSON value tokens (object and array): + * - JSON_OBJECT_BEGIN; + * - JSON_OBJECT_KEY; + * - JSON_OBJECT_END; + * - JSON_ARRAY_BEGIN; + * - JSON_ARRAY_END. + * + * Syntax-related tokens: + * - JSON_SYNTAX_WHITESPACE; + * - JSON_SYNTAX_OBJECT_COLON; + * - JSON_SYNTAX_OBJECT_COMMA; + * - JSON_SYNTAX_ARRAY_COMMA. + * + * And finally, the JSONReader-specific JSON_END indicating end-of-stream. + * + * \sa json_parse_syntax() + * \sa JSONReader::next() + */ + +/*************************************************************************/ + +/*! \class JSONParserOptions oes/core/utils/Json.h + * \brief Options for JSONReader. + */ + +/*! + * \brief Instantiates default options and limits for JSON parsing. + */ +JSONParserOptions::JSONParserOptions( +) : allow_scalar_(true) + , syntax_mode_(false) + , ignore_trailing_content_(false) + , max_depth_ (1<<8) // 256 + , max_array_size_ (1<<24) // 16M + , max_object_size_ (1<<16) // 64K + , max_string_byte_size_ (1<<24) // 16M + , max_objkey_byte_size_ (1<<16) // 64K +{} + +/*! \var JSONParserOptions::allow_scalar_ + * \brief Whether to allow parsing a primitive value on its one. + * + * Strict JSON requires a outer JSON object. + * This extension allows \em naked primitive values. + * + * Default to true. + */ + +/*! \var JSONParserOptions::syntax_mode_ + * \brief Whether to report whitespace and syntax related parsing events. + * + * Allows to reformat (indent, add or strip whitespace, etc...) JSON documents. + * + * Default to false. + */ + +/*! \var JSONParserOptions::ignore_trailing_content_ + * \brief Whether to continue parsing past the top-level object or array. + * + * If true, reports JSON_END as soon as the top-level object or array is + * \em closed (JSON_OBJECT_END or JSON_ARRAY_END), even if there is still + * textual content available. + * + * Default to false. + */ + +/*! \var JSONParserOptions::max_depth_ + * \brief Maximum nesting level (depth) allowed during parsing. + * + * Default to 256 levels. + */ + +/*! \var JSONParserOptions::max_array_size_ + * \brief Maximum number of values within an array. + * + * Default to 2^24 entries (around 16 millions). + */ + +/*! \var JSONParserOptions::max_object_size_ + * \brief Maximum number of key-value-pairs within an object. + * + * Default to 2^16 entries (around 64 thousands). + */ + +/*! \var JSONParserOptions::max_string_byte_size_ + * \brief Maximum raw byte size of a string primitive value. + * + * This value does not match the maximum character of the strings, because + * of UTF-8 encoding, and the use of escape characters (\\t, \\n, etc...) and + * unicode escapes, but the byte length of the string as it appears in the + * document (excluding the double-quotes required around the string). + * + * Default to 2^24 bytes (around 16 millions bytes). + */ + +/*! \var JSONParserOptions::max_objkey_byte_size_ + * \brief Maximum raw byte size of an object-key string value. + * + * This value does not match the maximum character of the strings, because + * of UTF-8 encoding, and the use of escape characters (\\t, \\n, etc...) and + * unicode escapes, but the byte length of the string as it appears in the + * document (excluding the double-quotes required around the string). + * + * Default to 2^16 bytes (around 64 thousands bytes). + */ + +/*************************************************************************/ + +/*! \internal + * + * Partial state of a JSONPullParser which can be cheaply captured and which + * JSONPullParser takes as explicit argument, to use the same code on either + * the parser's own (full) state, or a partial snapshot state, to efficiently + * implement peek(). + * + * FIXME: Introduce JSONTokenizer instead, for efficient peek()ing. + */ +struct JSONPullParserDynamicState { + const char* s = nullptr; + + JSONParsingEventType evt_ = JSON_END; + JSONRawStringRef text_; + + bool is_peek(const JSONPullParserDynamicState& other) const { + return this != &other; + } + + const char* set_event(JSONParsingEventType evt) { + evt_ = evt; + // Keep a prev event? + return nullptr; // no error + } +}; + +//! \internal Pull parser implementation behind JSONReader. +struct JSONPullParser : JSONPullParserDynamicState { + // Inherit from (partial) dynamic state, instead of composing it, + // to have less code to update. Should go away with JSONTokenizer. + using DynamicState = JSONPullParserDynamicState; + + std::weak_ptr parent_; + + JSONParserOptions options_; + + const char* uft8_ = nullptr; + const char* end = nullptr; + size_t len_ = 0; + + // TODO: Cleanup, move to header (as JSONFrame), + // and expose as JSONReader::backtrace() + struct Frame { + Frame(char type = 0) : type_(type), expect_key_(true), count_(0) {} + char type_; //!< the frame type, either array '[' or object '{' + bool expect_key_; // frames_; + + // WARNING: JSONSyntaxParser copy/paste's (with slight modifications) these + // methods, so changes here must be reflected there, and vice-versa. + bool eof(DynamicState& _); + bool consume_whitespace(DynamicState& _); + void consume_char(DynamicState& _, bool at_end_ok = false); + bool consume_unicode(); + + JSONParsingEventType next(DynamicState& _); + + const char* expect_value(DynamicState& _); + const char* expect_number(); + const char* expect_literal(DynamicState& _, const char* lit, int lit_len, JSONParsingEventType expected); + const char* expect_string(bool is_key); + const char* expect_array(); + const char* expect_array_content(DynamicState& _); + const char* expect_object(); + const char* expect_object_content(DynamicState& _); + + JSONParsingEventType verify(DynamicState& _, const char* err_msg); + void verify(bool condition, const char* err_msg); + void verify_event(JSONParsingEventType expected); + + void get_integer(int* i4_value, int64_t* i8_value, uint64_t* u8_value); +}; + +/*************************************************************************/ + +bool JSONPullParser::eof(DynamicState& _) { + return _.s >= end || consume_whitespace(_); +} + +JSONParsingEventType JSONPullParser::verify(DynamicState& _, const char* err_msg) { + if (err_msg) { + throw std::runtime_error(err_msg); + } + return _.evt_; +} + +void JSONPullParser::verify(bool condition, const char* err_msg) { + if (!condition) { + throw std::runtime_error(err_msg); + } +} + +void JSONPullParser::verify_event(JSONParsingEventType expected) { + verify(expected == evt_, "Incompatible type requested"); +} + +void JSONPullParser::consume_char(DynamicState& _, bool at_end_ok) { + if (++_.s >= end) { + verify(at_end_ok && _.s == end, "Unexpected end-of-value"); + } +} + +bool JSONPullParser::consume_whitespace(DynamicState& _) { + _.text_.reset(_.s); + const char*const prev_s = _.s; + for (unsigned char c = *_.s; _.s < end; c = *++_.s) { + if (c > 127 || !is_json_whitespace(c)) { + if (_.s > _.text_.beg_) { + _.text_.before_comma_colon_ = (*_.s == ',') || (*_.s == ':'); + _.text_.endsAt(_.s); + _.evt_ = JSON_SYNTAX_WHITESPACE; + // TODO: Syntax Mode + } + return false; + } + } + if (end > prev_s) { + // text_.before_comma_colon_ implicitly false from reset() + _.text_.endsAt(_.s); + _.evt_ = JSON_SYNTAX_WHITESPACE; + // TODO: Syntax Mode + } + return true; // EOF +} + +const char* JSONPullParser::expect_literal( + DynamicState& _, const char* lit, int lit_len, JSONParsingEventType expected +) { + if ((_.s + lit_len) > end) { + return "Unexpected end-of-value"; + } + if (0 != strncmp(_.s, lit, lit_len)) { + return "Invalid Value"; + } + _.text_.reset(_.s); + _.text_.endsAt(_.s += lit_len); + if (options_.ignore_trailing_content_ && frames_.empty()) { + end = s; // force JSON_END on next next() call + } + return _.set_event(expected); +} + +const char* JSONPullParser::expect_number() { + BOOST_ASSERT(*s == '-' || isdigit(*s)); + + text_.reset(s); + + if (*s == '-') { // optional + text_.num_.is_negative_ = true; + consume_char(*this); + } + + // required part + if (*s == '0') { + consume_char(*this, true); + if (s >= end) { + // just 0 is possible in naked 0 integer value. + text_.endsAt(s + 1); + if (options_.ignore_trailing_content_ && frames_.empty()) { + end = s; // force JSON_END on next next() call + } + return set_event(JSON_NUMBER); + } + // a leading 0 must be followed by a decimal-point, as leading + // zeros are not allowed in encoded JSON numbers. + if (*s != '.' && isdigit(*s)) { + return "Invalid number: Expected decimal-point"; + } + } else { + // expect_size_t + if (!isdigit(*s)) { + return "Invalid number: Expected digits"; + } + for (unsigned char c = *s; s < end; c = *++s) { + if (c < '0' || '9' < c) { + break; + } + } + } + + // optional fractional part (after the dot) + if (s < end && *s == '.') { + text_.num_.has_dot_ = true; + consume_char(*this); // must be followed by digits + // consume the digits that follow + const unsigned char first_c = *s; + if (first_c < '0' || '9' < first_c) { + return "Invalid number: expected decimals"; + } + consume_char(*this, true); // may have more digits + for (unsigned char c = *s; s < end; c = *++s) { + if (c < '0' || '9' < c) { + break; + } + } + } + + // optional exponential part + if (s < end && (*s == 'e' || *s == 'E')) { + text_.num_.has_exp_ = true; + consume_char(*this); // must be followed by +/- and/or digits + if (*s == '-' || *s == '+') { + consume_char(*this); // must be followed by digits + } + const unsigned char first_c = *s; + if (first_c < '0' || '9' < first_c) { + return "Invalid number: expected decimals"; + } + consume_char(*this, true); // may have more digits + for (unsigned char c = *s; s < end; c = *++s) { + if (c < '0' || '9' < c) { + break; + } + } + } + + text_.endsAt(s); + if (options_.ignore_trailing_content_ && frames_.empty()) { + end = s; // force JSON_END on next next() call + } + return set_event(JSON_NUMBER); +} + +void JSONPullParser::get_integer(int* i4_value, int64_t* i8_value, uint64_t* u8_value) { + // Get the int part. + const bool is_negative = text_.num_.is_negative_; + const char* beg = is_negative? text_.beg_ + 1: text_.beg_; + const char* tmp_end = text_.end_; + if (text_.num_.has_dot_) { + tmp_end = text_.find('.'); + BOOST_ASSERT(tmp_end != text_.end_); + } else if (text_.num_.has_exp_) { + tmp_end = text_.find('e'); + if (tmp_end == text_.end_) { + tmp_end = text_.find('E'); + } + BOOST_ASSERT(tmp_end != text_.end_); + } + + const uint64_t int_part = parser_integer(beg, tmp_end); + + if (!text_.num_.has_dot_ && !text_.num_.has_exp_) { + // naked integer value + if (int_part <= (uint64_t)std::numeric_limits::max()) { + const int as_int32 = static_cast(int_part); + const int value = is_negative? -as_int32: as_int32; + if (i4_value) { *i4_value = value; return; } + if (i8_value) { *i8_value = value; return; } // widening + if (u8_value) { verify(!is_negative, "Sign mismatch"); *u8_value = int_part; return; } + + } else if (is_negative && int_part == (1 + (uint64_t)std::numeric_limits::max())) { + // smallest int, i.e. -2,147,483,648 does not fit in absolute value + // as an int, so we cannot cast int_part into as_int32 as above. + const int value = std::numeric_limits::min(); + if (i4_value) { *i4_value = value; return; } + if (i8_value) { *i8_value = value; return; } // widening + verify(false, "Cannot get negative integer as unsigned"); + + } else if (int_part <= (uint64_t)std::numeric_limits::max()) { + const int64_t as_int64 = static_cast(int_part); + const int64_t value = is_negative? -as_int64: as_int64; + if (i4_value) { verify(false, "Integer Overflow"); return; } + if (i8_value) { *i8_value = value; return; } + if (u8_value) { verify(!is_negative, "Sign mismatch"); *u8_value = int_part; return; } + + } else if (is_negative && int_part == (1 + (uint64_t)std::numeric_limits::max())) { + // smallest int64, i.e. -9,223,372,036,854,775,808 does not fit in absolute value + // as an int64, so we cannot cast int_part into as_int64 as above. + const int64_t value = std::numeric_limits::min(); + if (i4_value) { verify(false, "Integer Overflow"); return; } + if (i8_value) { *i8_value = value; return; } + verify(false, "Cannot get negative integer as unsigned"); + + } else if (!is_negative && int_part <= std::numeric_limits::max()) { + if (i4_value) { verify(false, "Integer Overflow"); return; } + if (i8_value) { verify(false, "Integer Overflow"); return; } + if (u8_value) { *u8_value = int_part; return; } + + } else { + if (is_negative) { + BOOST_ASSERT(int_part > (uint64_t)std::numeric_limits::max() + 1); + verify(false, "Numeric overflow: < MIN_SSIZE_T"); + } else { + // we already throw in parse_number(), so should never be reached + verify(false, "Numeric overflow: > MAX_SIZE_T"); + } + } + } +} + +// only verifies the codepoint, does not extract its value +bool JSONPullParser::consume_unicode() { + BOOST_ASSERT(*s == 'u'); + verify((s + 4) < end, "Truncated Unicode Codepoint"); + verify( + isxdigit(*++s) && isxdigit(*++s) && + isxdigit(*++s) && isxdigit(*++s), + "Invalid Unicode Codepoint" + ); + return text_.str_.has_unicode_ = true; +} + +// positions \a s one-char past the closing double-quote +const char* JSONPullParser::expect_string(bool is_key) { + BOOST_ASSERT(*s == '"'); + text_.reset(s); + consume_char(*this); + const size_t max_len = is_key? + options_.max_objkey_byte_size_: options_.max_string_byte_size_; + // Note: +1 below to account for terminating quote + const char*const s_end = std::min<>(s + max_len + 1, end); + for (unsigned char c = *s; s < s_end; c = *++s) { + if (c > 127) { + // part of a multi-byte UTF-8 encoded string. + // TODO: Check valid multi-byte. + text_.str_.not_ascii_ = true; + continue; + } + switch (c) { + case '"': { + // Reached the end of the string. + consume_char(*this, true); + text_.endsAt(s); + if (!is_key && options_.ignore_trailing_content_ && frames_.empty()) { + end = s; // force JSON_END on next next() call + } + return set_event(is_key? JSON_OBJECT_KEY: JSON_STRING); + } + case '\\': + if ((s + 1) >= s_end) { + if (s_end < end) { + return "JSON maximum string length exceeded"; + } else { + return "Unexpected Partial Escape"; + } + } + switch (c = *++s) { + case 'u': // encoded Unicode codepoint must be followed by 4 hex + consume_unicode(); + break; + + case '"': case '\\': case '/': + text_.str_.has_escape_ = true; + break; + + case 'b': case 'f': case 'n': case 'r': case 't': + text_.str_.has_escape_ = true; + break; + + default: // only above characters are allowed after backslash + return "Invalid String: Unexpected Escape"; + } + break; + + case '/': + // Even though forward-slash can be escaped, as per the spec, + // (and JSONWriter always escape it), it is also valid to have an + // unescaped one in an encoded JSON string. See Errata ID: 3159 + // from http://www.rfc-editor.org/errata_search.php?rfc=4627 + //return "Invalid String: Unescaped forward-slash"; + break; + + default: + if (!isprint(c)) { + // should have been encoded in range \u0000-\u001f + return "Invalid String: Unencoded non-printable character"; + } + } + } + if (s_end < end) { + return "JSON maximum string length exceeded"; + } + // reached the end without seeing the closing double-quote + return "Invalid String: No closing double-quote"; +} + +const char* JSONPullParser::expect_array() { + BOOST_ASSERT(*s == '['); + verify(frames_.size() < options_.max_depth_, "JSON maximum depth exceeded"); + frames_.push_back('['); + text_.reset(s).endsAt(s + 1); + consume_char(*this); + return set_event(JSON_ARRAY_BEGIN); +} + +const char* JSONPullParser::expect_array_content(DynamicState& _) { + Frame& frame = frames_.back(); + BOOST_ASSERT('[' == frame.type_); + + const bool peeking = _.is_peek(*this); + + verify(_.s < end, "Unexpected end-of-array"); + consume_whitespace(_); + if (*_.s == ']') { + if (!peeking) { + frames_.pop_back(); + } + _.text_.reset(_.s).endsAt(_.s + 1); + consume_char(_, true); + if (!peeking && options_.ignore_trailing_content_ && frames_.empty()) { + end = _.s; // force JSON_END on next next() call + } + return _.set_event(JSON_ARRAY_END); + } + + const bool first = (frame.count_ == 0); + // If we expect a comma + if (!first) { + if (*_.s == ',') { + // TODO: Syntax Mode + //handler_(JSON_SYNTAX_ARRAY_COMMA, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(_); + } else { + return "Invalid array: missing comma separator"; + } + } + + verify( + frame.count_ < options_.max_array_size_, + "JSON maximum array entry count exceeded" + ); + if (!peeking) { + ++frame.count_; + } + return expect_value(_); +} + +const char* JSONPullParser::expect_object() { + BOOST_ASSERT(*s == '{'); + verify(frames_.size() < options_.max_depth_, "JSON maximum depth exceeded"); + frames_.push_back('{'); + text_.reset(s).endsAt(s + 1); + consume_char(*this); + return set_event(JSON_OBJECT_BEGIN); +} + +const char* JSONPullParser::expect_object_content(DynamicState& _) { + Frame& frame = frames_.back(); + BOOST_ASSERT('{' == frame.type_); + + const bool peeking = _.is_peek(*this); + + verify(_.s < end, "Unexpected end-of-object"); + consume_whitespace(_); + if (*_.s == '}') { + verify(frame.expect_key_ == true, "Unexpected end-of-object"); + if (!peeking) { + frames_.pop_back(); + } + _.text_.reset(_.s).endsAt(_.s + 1); + consume_char(_, true); + if (!peeking && options_.ignore_trailing_content_ && frames_.empty()) { + end = _.s; // force JSON_END on next next() call + } + return _.set_event(JSON_OBJECT_END); + } + + const bool first = (frame.count_ == 0); + // If we expect a comma + if (!first && frame.expect_key_) { + if (*_.s == ',') { + // TODO: Syntax Mode + //handler_(JSON_SYNTAX_OBJECT_COMMA, text_.reset(s).endsAt(s + 1), user_data_); + consume_char(_); + consume_whitespace(_); + } else { + return "Invalid object: missing comma separator"; + } + } + + if (frame.expect_key_) { + verify(*_.s == '"', "Missing expected object key"); + if (peeking) { + return _.set_event(JSON_OBJECT_KEY); + } + frame.expect_key_ = false; + return expect_string(true); + } else { + verify(*_.s == ':', "Missing expected object key:value separator"); + _.text_.reset(_.s); + consume_char(_); + // TODO: Syntax Mode + //handler_(JSON_SYNTAX_OBJECT_COLON, text_.endsAt(s), user_data_); + + verify( + frame.count_ < options_.max_object_size_, + "JSON maximum object key-value-pair count exceeded" + ); + if (!peeking) { + ++frame.count_; + frame.expect_key_ = true; + } + return expect_value(_); + } +} + +const char* JSONPullParser::expect_value(DynamicState& _) { + if (consume_whitespace(_)) { + return "Unexpected end-of-value"; + } + + const bool peeking = _.is_peek(*this); + + switch (*_.s) { + case '{': + return peeking? _.set_event(JSON_OBJECT_BEGIN): expect_object(); + + case '[': + return peeking? _.set_event(JSON_ARRAY_BEGIN): expect_array(); + + case '"': + return peeking? _.set_event(JSON_STRING): expect_string(false); + + case '-': // case '+': leading plus not allowed by JSON spec + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return peeking? _.set_event(JSON_NUMBER): expect_number(); + + case 't': + return expect_literal(_, "true", 4, JSON_TRUE); + case 'f': + return expect_literal(_, "false", 5, JSON_FALSE); + case 'n': + return expect_literal(_, "null", 4, JSON_NULL); + } + + return "Invalid JSON value: Unexpected character"; +} + +JSONParsingEventType JSONPullParser::next(DynamicState& _) { + const char state = frames_.empty()? 0: frames_.back().type_; + + switch (state) { + case 0 : return eof(_)? _.evt_ = JSON_END: // force evt_ to END, for current() + verify(_, expect_value(_)); + case '[': return verify(_, expect_array_content(_)); + case '{': return verify(_, expect_object_content(_)); + } + + verify(false, "Unknown error"); + return JSON_END; // unreachable +} + +/*************************************************************************/ + +struct JSONReader::SavePointImpl { + const JSONReader* owner_; + JSONPullParser parser_; +}; + +/*************************************************************************/ + +/*! \class JSONReader oes/core/utils/Json.h + * \brief Reads (parses) a JSON document. + * + * JavaScript Object Notation (JSON) is a text format for the + * serialization of structured data. It is derived from the object + * literals of JavaScript, as defined in the ECMAScript Programming + * Language Standard, Third Edition. + * + * JSON can represent four primitive types (strings, numbers, booleans, + * and null) and two structured types (objects and arrays). + * + * A string is a sequence of zero or more Unicode characters [UNICODE]. + * + * An object is an unordered collection of zero or more name/value + * pairs, where a name is a string and a value is a string, number, + * boolean, null, object, or array. + * + * An array is an ordered sequence of zero or more values. + * + * JSONReader will \em incrementally parse the document, each time its next() + * method is called, returning the type of JSON value seen, and providing + * accessors to extract typed C/C++ values corresponding to those JSON values. + * + * JSONReader's API mimics JDK 7's javax.json.stream.JsonParser, except its + * next() method returns JSON_END instead of having an has_next() method. + * + * JSONReader supports a limited number of options to customize its behavior + * and in particular a set of limits to how large values can be. + * + * For example, the JSON document {"version":"1.2.3","values":[1,2.3,true,null,{}]} + * will generate the following parsing events when passed to parse_json() below: + * \code + * void parse_json(const std::string& doc) { + * JSONReader json(doc.c_str(), doc.size()); + * json.next() // ==> JSON_OBJECT_BEGIN + * // 1st key-value pair + * json.next() // ==> JSON_OBJECT_KEY + * json.get_string() // ==> std::string("version") + * json.next() // ==> JSON_STRING + * json.get_string() // ==> std::string("1.2.3") + * // 2nd key-value pair + * json.next() // ==> JSON_OBJECT_KEY + * json.get_string() // ==> std::string("values") + * json.next() // ==> JSON_ARRAY_BEGIN + * // array content + * json.next() // ==> JSON_NUMBER + * json.is_integral() // ==> true + * json.get_int() // ==> 1 + * json.next() // ==> JSON_NUMBER + * json.is_integral() // ==> false + * json.get_int() // ==> 2.3 + * json.next() // ==> JSON_TRUE + * json.get_boolean() // ==> true + * json.next() // ==> JSON_NULL + * json.next() // ==> JSON_OBJECT_BEGIN + * json.next() // ==> JSON_OBJECT_END + * + * json.next() // ==> JSON_ARRAY_END + * json.next() // ==> JSON_OBJECT_END + * json.next() // ==> JSON_END + * } + * \endcode + * + * JSONReader is often called a \em PULL or \em STREAMING parser, because + * client code \em pulls events from the parser, as opposed to using + * json_parse(const char*,size_t,JSONHandler&) which uses a \em PUSH parser + * which calls back on a client-code-provided \em handler. A push parser + * decides which client-code handler method to call (handle_number(int) vs + * handle_number(uint64_t)), whereas control is reversed with JSONReader and + * its pull-parsing model, the client-code decide how which number-based + * method to call, and can even access the token() directly. + * + * Pull-parsing makes it easy to nest parsing code, by delegating parsing of + * a JSON document piece (typically an object) to another method, thus writing + * Recursive Decent Parsers with ease. + * + * \note This reader allows reading a single \em primitive value (either a + * number, a string, a boolean, or null) by itself, outside of an array + * or an object, as an extension, when JSONParserOptions::allow_scalar_ + * is true (the default). + * + * \sa http://tools.ietf.org/html/rfc4627 + * \sa http://docs.oracle.com/javaee/7/api/javax/json/stream/JsonParser.html + */ + +/*! + * \brief Instantiates a new JSON reader (pull parser). + * + * \param json_utf8_text UTF-8 encoded text of the JSON document to read. + * \param len the byte-length of the text in \a json_utf8_text. + * \param options the parsing options and limits. + */ +JSONReader::JSONReader( + const char* json_utf8_text, size_t len, + const JSONParserOptions& options +) : impl_(new JSONPullParser) +{ + impl_->uft8_ = json_utf8_text; + impl_->len_ = len; + + impl_->s = json_utf8_text; + impl_->end = json_utf8_text + len; + + impl_->options_ = options; +} + +//! \internal Instantiates nested parser +JSONReader::JSONReader( + const std::shared_ptr& outer_parser +) : impl_(new JSONPullParser) +{ + BOOST_ASSERT(outer_parser->s <= outer_parser->text_.end_); + + // Do a peek on outer_parser. Note that we are not doing switch (peek()) + // in JSONReader::scope() because we also need the token to advance the + // nested parser to the start of the next value. + JSONPullParserDynamicState state(*outer_parser); // truncation on purpose + JSONParsingEventType next_evt = outer_parser->next(state); + switch (next_evt) { + case JSON_NULL: + case JSON_TRUE: + case JSON_FALSE: + case JSON_STRING: + case JSON_NUMBER: + case JSON_OBJECT_BEGIN: + case JSON_ARRAY_BEGIN: + break; + + default: + throw std::runtime_error( + "JSONReader: must expect JSON value on scope()" + ); + } + + impl_->uft8_ = state.text_.beg_; + impl_->parent_ = outer_parser; + impl_->len_ = (outer_parser->end - impl_->uft8_); + + impl_->s = impl_->uft8_; + impl_->end = outer_parser->end; + + impl_->options_ = outer_parser->options_; + impl_->options_.ignore_trailing_content_ = true; + impl_->options_.max_depth_ -= outer_parser->frames_.size(); +} + +/*! + * \brief Deletes this JSON reader. + */ +JSONReader::~JSONReader() { + if (std::shared_ptr parent = impl_->parent_.lock()) { + // update parent's "partial" state with the one of the nested parser + (JSONPullParserDynamicState&)(*parent) = (JSONPullParserDynamicState&)(*impl_); + + if (!impl_->frames_.empty()) { // most likely on error + parent->frames_.insert( + parent->frames_.end(), + impl_->frames_.begin(), impl_->frames_.end() + ); + } else if (!parent->frames_.empty()) { + // FIXME: This "patches up" the frame a posteriori, and the fact + // that it is necessary is not a good sign... + JSONPullParser::Frame& frame = parent->frames_.back(); + switch (frame.type_) { + case '{': + frame.expect_key_ = true; + // fall-thru + case '[': + ++frame.count_; + break; + + default: + OES_DEBUG_ASSERT(false); + } + } + } + impl_.reset(); +} + +/*! + * \brief Tests the type of the token + * \param expected expected token type + * \param received received token type + */ +bool JSONReader::test_token_type(JSONParsingEventType expected, JSONParsingEventType received) { + if (expected != received) { + return false; + } + return true; +} + +/*! + * \brief Tests the token string + * \param expected expected string + * \param received received string + */ +bool JSONReader::test_token_string(std::string expected, std::string received) { + if (expected != received) { + return false; + } + return true; +} + +/*! + * \brief Gets the type for the next parsing state, + * without \em advancing this reader. + * + * This method behaves exactly as next() (i.e. can throw exceptions on JSON + * syntax errors), except it does not modify the parsing state at all. + * + * \return JSON_END if positioned at the end of the document to parse; + * the token type (parsing event type) otherwise. + * \throws std::runtime_error on invalid JSON syntax, or when a parsing + * limit is exceeded. + * + * \note peek() is similar to a \em 1-event savepoint, i.e. it is equivalent + * to creating a \em savepoint, calling next() and recording the result, + * then call rollback_to(savepoint). + */ +JSONParsingEventType JSONReader::peek() const { + JSONPullParserDynamicState state(*impl_); // truncation on purpose + return impl_->next(state); +} + +/*! + * \brief Gets the type for the next parsing state. + * + * JSONReader keeps track of parsing states to determine which valid parsing + * state(s) can follow the previous one. + * + * \return JSON_END if positioned at the end of the document to parse; + * the token type (parsing event type) otherwise. + * \throws std::runtime_error on invalid JSON syntax, or when a parsing + * limit is exceeded. + */ +JSONParsingEventType JSONReader::next() { + return impl_->next(*impl_); +} + +/*! + * \brief Gets the type for the current parsing state. + * + * Unlike next(), current() does not \em advance to the next token, but instead + * returns the same event than the previous next() call returned; + * + * \return what the previous call to next() returned. If previous next() threw + * a C++ exception, the result is undefined and should not be depended on. + * + * \sa next() +*/ +JSONParsingEventType JSONReader::current() { + return impl_->evt_; +} + +/*! + * \brief Creates a nested JSONReader, to \em isolate reading a \em child value. + * + * Parsing non-trivial JSON documents typically involves calling json_parse() + * helper methods to delegate parsing subsets of a document, typically + * corresponding to a given type. For example, parsing a color-map implemented + * in C++ as a std::map where MyColor is a struct made of + * the RGB components of the color can be represented in JSON as + * \code + * { + * "white": { "r": 1.0, "g": 1.0, "b": 1.0 } + * "cyan": { "r": 0.0, "g": 1.0, "b": 1.0 } + * ... + * } + * \endcode + * + * Instead of writing the JSON writing/parsing code for MyColor in the color-map + * parsing code, one can delegate parsing the color to + * json_parse(JSONReader& json, MyColor& color), to be reused by color-map and + * any other client code that deals in MyColor. But when delegating such parsing + * to other code, one runs the risk of the json_parse() method reading too many + * JSON events/tokens (i.e. making too many next() calls), stepping out of the + * JSON \b value being read. The scope() method creates a special \em nested + * JSONReader instance, still connected to its \em parent (outer) JSONReader, + * which won't allow the parsing code to \em overstep the JSON \em boundaries + * of the value being read. (it returns JSON_END as soon as the value ends). + * Once the called code returns, the nested parser must be deleted to resume + * parsing on the outer/parent JSONReader, either explicitly via .reset(): + * \code + * JSONReader json(...); + * std::map colormap; + * json.next(); // ==> JSON_OBJECT_BEGIN + * while (json.peek() != JSON_OBJECT_END) { + * json.next(); // ==> JSON_OBJECT_KEY + * std::string name = json.get_key(); + * MyColor c; + * JSONReader::Scope nested(json.scope()); + * const bool ok = json_parse(*nested, c); // check ok + * nested.reset(); + * } + * \endcode + * or implicitly if used inline: + * \code + * ... + * while (json.peek() != JSON_OBJECT_END) { + * ... + * MyColor c; + * const bool ok = json_parse(*json.scope(), c); // check ok + * } + * \endcode + * (.scope() returns an std::shared which remains \em alive for the duration of + * the statement, until the semicolon when it is implicitly deleted, \em closing + * the nested scope/reader). + * + * A scope can only be created when expecting a JSON value, i.e. when reading + * array values, or the value after an object key. + * + * \return A nested JSONReader. + * \throw std::runtime_error when this JSONReader's is not expected a JSON + * value next. + * + * \warning a scope \b must be deleted before calling methods of the parent + * reader again + */ +JSONReader::Scope JSONReader::scope() { + return JSONReader::Scope(new JSONReader(impl_)); +} + +/*! + * \brief Takes a \em snapshot of the reader, to possibly go back to. + * + * This is similar to a SQL transaction savepoint, that you can rollback_to() + * in case an error occurs after the savepoint was taken, to attempt or retry + * to complete the \em transaction (JSON parsing in this case) successfully. + * + * \return the savepoint on success; null otherwise. + */ +JSONReader::SavePoint JSONReader::savepoint() { + // Not sure whether we should enforce specific points in the + // JSON document when savepoints can be taken or not... + return SavePoint(new SavePointImpl { this, *impl_ }); +} + +/*! + * \brief Rollbacks this parser to a previous saved state. + * + * Savepoints are used to recover from parsing failures, typically in nested + * parsing code. One creates a JSONSavePoint before entering the nested parsing + * code, and on failure of that code, rollback_to() the savepoint, and then + * call skip_next() for example in an attempt to parse the invalid \em value + * and go to the next value. + * + * \param savepoint the previously created savepoint, typically on the stack. + * \return true on a successful rollback; false otherwise, if \sa savepoint + * was not created from this parser, or we can no longer go back to the + * savepoint because this parser advanced past the array or object the + * savepoint was created in. + * + * \sa peek(), an implicit \em 1-event savepoint. + */ +bool JSONReader::rollback_to(const SavePoint& savepoint) { + OES_ASSERT(savepoint, return false;); + OES_ASSERT(savepoint->owner_ == this, return false;); + OES_ASSERT(savepoint->parser_.frames_.size() <= depth(), return false;); + // TODO: check savepoint's frames_ are a prefix to this frames_ + *impl_ = savepoint->parser_; + return true; // savepoint deleted here +} + +/*! + * \brief Skips the next JSON value. + * + * Convenience method equivalent to \code next(); skip_current() \endcode. + * See skip_current() for details. + * + * \return the number of next() calls made to skip the value. + * \note this method throws on any JSON syntax error, like next() does. + */ +size_t JSONReader::skip_next() { + next(); + return skip_current(); +} + +/*! + * \brief Skips the current JSON value. + * + * Scalar values do not call next(), and return one. + * + * Object/array values \b must be on their opening event (JSON_OBJECT_BEGIN + * and JSON_ARRAY_BEGIN respectively), and are skipped in full, irrespective + * of content (even nested content). + * + * \return the number of next() calls made to skip the value, + * plus one for the current event. + * \throws if next() throws on syntax errors, or the current event is neither + * a scalar value (an object-key is not considered scalar), nor the beginning + * of an object or array. + */ +size_t JSONReader::skip_current() { + // TODO: Syntax mode + switch (current()) { + case JSON_END: + return 0; + + case JSON_NULL: + case JSON_TRUE: + case JSON_FALSE: + case JSON_STRING: + case JSON_NUMBER: + return 1; + + case JSON_OBJECT_BEGIN: { + const size_t curr_depth = depth(); + size_t count = 2; + for (next(); depth() >= curr_depth; next()) ++count; + impl_->verify_event(JSON_OBJECT_END); + return count; + } + + case JSON_ARRAY_BEGIN: { + const size_t curr_depth = depth(); + size_t count = 2; + for (next(); depth() >= curr_depth; next()) ++count; + impl_->verify_event(JSON_ARRAY_END); + return count; + } + + default: + impl_->verify(false, "Cannot skip partial JSON value"); + return 0; + } +} + +/*! + * \brief Gets the current token. + * + * After JSONReader properly identified the token type, as returned by next(), + * client code typically calls one of the get_*() methods, but can also access + * the raw text for the token and interpret it as a integer or double or + * boolean itself. This can be used for example to get an integer larger than + * uint64_t, and store it in some \em BigNum class specific to the client code. + * + * \return the raw text of the current parsing state, i.e. token. + * + * \note invalid to call after next() threw an std::runtime_error + * or returned JSON_END. + */ +JSONToken JSONReader::token() { + return impl_->text_; // truncation on purpose +} + +/*! + * \brief How many nested object or arrays did we parse so far. + * + * Increments each time we encounter a JSON_OBJECT_BEGIN or JSON_ARRAY_BEGIN + * event; Similarly decrements on every JSON_OBJECT_END or JSON_ARRAY_END. + * + * \return the current object/array parsing depth. + */ +size_t JSONReader::depth() { + return impl_->frames_.size(); +} + +/*! + * \brief How many object key-value pair or array values read so far, + * at the current depth. + * + * Every time depth() changes value, count() also likely changes, since count() + * is depth()-dependent. Resets to zero after JSON_OBJECT_BEGIN or + * JSON_ARRAY_BEGIN, and restores the previous depth's count (+ 1) after + * JSON_OBJECT_END or JSON_ARRAY_END. + * + * \return the number of \em values in the current object/array read so far. + */ +size_t JSONReader::count() { + return impl_->frames_.empty()? (impl_->eof(*impl_)? 1: 0): impl_->frames_.back().count_; +} + +/*! + * \brief Determines whether the current token is an integral number. + * + * Note that JSONReader supports large numbers at parsing, but is limited + * to C/C++ datatypes for returning numbers. Access token() directly, and + * convert it to some \em BigNum class in client-code to overcome this + * limitation. + * + * \return true if integral; false otherwise (i.e. floating-point). + * \throws std::runtime_error when previous next() did not return JSON_NUMBER, + * or is_integral() is false. + * + * \note It is not an error to call get_float() or get_double() for integral + * values, but can result in roundoff approximations for large integers; + * + * \sa get_int() + * \sa get_int64_t() + * \sa get_uint64_t() + */ +bool JSONReader::is_integral() { + impl_->verify_event(JSON_NUMBER); + return !impl_->text_.num_.has_exp_ && !impl_->text_.num_.has_dot_; +} + +/*! + * \brief Gets the current integral number as a C/C++ int. + * + * \return the int value. + * \throws std::runtime_error when previous next() did not return JSON_NUMBER, + * or is_integral() is false, or the integral values overflows an int. + * + * \sa is_integral() + * \sa token() + */ +int JSONReader::get_int() { + impl_->verify(is_integral(), "Not an integer"); + int value = 0; + impl_->get_integer(&value, nullptr, nullptr); + return value; +} + +/*! + * \brief Gets the current integral number as a C/C++ int64_t. + * + * \return the int64_t value. + * \throws std::runtime_error when previous next() did not return JSON_NUMBER, + * or is_integral() is false, or the integral values overflows an int. + * + * \sa is_integral() + * \sa token() + */ +int64_t JSONReader::get_int64_t() { + impl_->verify(is_integral(), "Not an integer"); + int64_t value = 0; + impl_->get_integer(nullptr, &value, nullptr); + return value; +} + +/*! + * \brief Gets the current integral number as a C/C++ uint64_t. + * + * \return the uint64_t value. + * \throws std::runtime_error when previous next() did not return JSON_NUMBER, + * or is_integral() is false, or the integral values overflows an uint64_t. + * + * \sa is_integral() + * \sa token() + */ +uint64_t JSONReader::get_uint64_t() { + impl_->verify(is_integral(), "Not an integer"); + uint64_t value = 0; + impl_->get_integer(nullptr, nullptr, &value); + return value; +} + +/*! + * \brief Gets the current number as a C/C++ float. + * + * \return the float value (possibly integral). + * \throws std::runtime_error when previous next() did not return JSON_NUMBER; + * Or the conversion overflows a float, as per C++ I/O Stream semantic. + * + * \note Text-to-Floating-Point conversion uses C++ I/O Stream, but using + * std::istream::operator>>(float&). Client code can perform its own + * conversion by accessing the token() directly. + * + * \sa is_integral() + * \sa token() + */ +float JSONReader::get_float() { + impl_->verify_event(JSON_NUMBER); + return parse_number(impl_->text_.beg_, impl_->text_.end_); +} + +/*! + * \brief Gets the current number as a C/C++ double. + * + * \return the double value (possibly integral). + * \throws std::runtime_error when previous next() did not return JSON_NUMBER; + * Or the conversion overflows a float, as per C++ I/O Stream semantic. + * + * \note Text-to-Floating-Point conversion uses C++ I/O Stream, but using + * std::istream::operator>>(double&). Client code can perform its own + * conversion by accessing the token() directly. + * + * \sa is_integral() + * \sa token() + */ +double JSONReader::get_double() { + impl_->verify_event(JSON_NUMBER); + return parse_number(impl_->text_.beg_, impl_->text_.end_); +} + +/*! + * \brief Gets the current string, possibly decoded. + * + * \return the string value (possibly an object key). + * \throws std::runtime_error when previous next() did not return + * JSON_STRING or JSON_OBJECT_KEY. + * + * \warning neither next() or get_string(), in case of JSON_OBJECT_KEY, + * check the key's unicity within the current object. Client code + * must test this unicity itself, if required. + * + * \sa token() + */ +std::string JSONReader::get_string() { + impl_->verify( // TODO: accept JSON_STRING only + impl_->evt_ == JSON_STRING || impl_->evt_ == JSON_OBJECT_KEY, + "Incompatible type requested" + ); + std::string buffer; + buffer.reserve(impl_->text_.size()); + impl_->text_.decode(buffer, true); + return buffer; +} + +/*! + * \brief Gets the current string, accepting a null literal. + * + * Convenience method returning an empty string on a null literal, + * and get_string() otherwise. + * + * \sa JSONWriter::string(const std::string&, bool) + */ +std::string JSONReader::get_string_or_null() { + if (impl_->evt_ == JSON_NULL) { + return std::string(); + } + return get_string(); +} + +/*! + * \brief Gets the current JSON_TRUE or JSON_FALSE as a C/C++ bool. + * + * Convience method, since the token type itself carries the value. + * + * \return true when next() returned JSON_TRUE; + * false when it returned JSON_FASLE. + * \throws std::runtime_error when previous next() + * did not return JSON_TRUE or JSON_FALSE. + * + * \note true and false are not considered \em integral. + * + * \sa is_integral() + */ +bool JSONReader::get_boolean() { + impl_->verify( + impl_->evt_ == JSON_TRUE || impl_->evt_ == JSON_FALSE, + "Incompatible type requested" + ); + return (impl_->evt_ == JSON_TRUE)? true: false; +} + +/*! + * \brief Gets the current \em decoded object key. + * + * This method must return the key by value, in case the raw JSON text uses + * escape sequences or unicode characters. For simple pure-ASCII keys, prefer + * is_key() which can avoid the key copy when no decoding is necessary. + * + * \return the object key. + * \throws std::runtime_error when previous next() did not return JSON_OBJECT_KEY. + * + * \warning neither next() or get_string(), in case of JSON_OBJECT_KEY, + * check the key's unicity within the current object. Client code + * must test this unicity itself, if required. + * + * \sa is_key() + * \sa JSONToken::matches_unquote() + */ +std::string JSONReader::get_key() { + impl_->verify_event(JSON_OBJECT_KEY); + std::string buffer; + buffer.reserve(impl_->text_.size()); + impl_->text_.decode(buffer, true); + return buffer; +} + +/*! + * \brief Convenience method equivalent to is_key(key, strlen(key)). + * + * \param key the null-terminated key to test against. + * \return true when \a key is the current object key; false otherwise. + * \throws std::runtime_error when previous next() did not return JSON_OBJECT_KEY. + */ +bool JSONReader::is_key(const char* key) { + return is_key(key, strlen(key)); +} + +/*! + * \brief Tests whether the current object key matches a given string literal. + * + * For simple pure-ASCII keys, this method is faster than get_key() == key. + * + * \param key the key (not necessarily null-terminated) key to test against. + * \param len the key len. + * \return true when \a key is the current object key; false otherwise. + * \throws std::runtime_error when previous next() did not return JSON_OBJECT_KEY. + * + * \sa get_key() + * \sa JSONToken::matches_unquote() + */ +bool JSONReader::is_key(const char* key, size_t len) { + impl_->verify_event(JSON_OBJECT_KEY); + if (impl_->text_.str_.not_ascii_ || + impl_->text_.str_.has_escape_ || + impl_->text_.str_.has_unicode_ + ) { + return get_key() == key; + } + return impl_->text_.matches_unquote(key, len); +} + +/*************************************************************************/ + +/*! + * \brief Validates a JSON document is well-formed. + * + * \param json_utf8_text UTF-8 encoded text of the JSON document. + * \param len the byte-length of the text in \a json_utf8_text. + * \return true if \a json_utf8_text was entirely consumed by this parser, + * with leading and trailing JSON whitespace ignored, and the text in + * between was a valid well-formed JSON document; false otherwise. + * + * \note Does not check number underflow/overflow, only that the string + * representation is consistent with a number. + */ +bool json_validate_syntax(const char* json_utf8_text, size_t len) { + try { + JSONSyntaxParser parser(do_nothing_handler); + return 0 == parser.parse(json_utf8_text, len); + } catch (const std::runtime_error&) { + return false; + } +} + +/*************************************************************************/ + +namespace { + +struct FormatHandler { + std::ostream& os_; + std::string key_sep_; + std::vector indents_; // FIXME: Use C++11 std::array instead + std::string indentbuf_; + std::string* curr_indent_; + JSONParsingEventType prev_evt_; + int level_; + + FormatHandler( + std::ostream& os, + const std::string& indent = "\t", + const std::string& key_sep = ": " + ) : os_(os) + , key_sep_(key_sep) + , curr_indent_(nullptr) + , prev_evt_(JSON_SYNTAX_WHITESPACE) + , level_(0) + { + // check valid indent + for (char c : indent) { + if (!is_json_whitespace(c)) { + throw std::runtime_error("Invalid JSON indent: not spaces"); + } + } + + // check valid key/value separator + bool hasColon = false; + for (char c : key_sep) { + if (c == ':') { + if (hasColon) { + throw std::runtime_error("Invalid JSON key/value separator"); + } + hasColon = true; + } else if (!is_json_whitespace(c)) { + throw std::runtime_error("Invalid JSON key/value separator"); + } + } + if (!hasColon) { + throw std::runtime_error("Invalid JSON key/value separator"); + } + + // pre-compute some indents + indents_.reserve(16); + indents_.push_back(""); + indents_.push_back(indent); + for (size_t i = 2; i < indents_.capacity(); ++i) { + // FIXME: use C++11 emplace_back() instead of push_back() + indents_.push_back(indents_[i - 1] + indent); + } + curr_indent_ = &indents_[0]; + } + + void update_indent() { + if (level_ < static_cast(indents_.size())) { + curr_indent_ = &indents_[level_]; + } else { + indentbuf_.clear(); + indentbuf_.reserve(level_ * indents_[1].size()); + size_t remaining = level_ + 1; + for (; remaining >= indents_.size(); remaining -= indents_.size()) { + indentbuf_.append(indents_.back()); + } + if (remaining) { + indentbuf_.append(indents_[remaining]); + } + curr_indent_ = &indentbuf_; + } + } + + void operator()(JSONParsingEventType evt, const JSONRawStringRef& text) { + // adjust the level + switch (evt) { + case JSON_SYNTAX_WHITESPACE: + return; // ignore existing whitespace + + case JSON_SYNTAX_OBJECT_COLON: + os_ << key_sep_; + return; + + case JSON_OBJECT_BEGIN: + case JSON_ARRAY_BEGIN: + if (prev_evt_ != JSON_SYNTAX_OBJECT_COLON) { + os_ << *curr_indent_; + } + os_ << text << '\n'; + ++level_; + update_indent(); + return; + + case JSON_OBJECT_END: + case JSON_ARRAY_END: + // FIXME: Special case empty array/object, avoiding indent? + --level_; + update_indent(); + if (prev_evt_ != JSON_OBJECT_BEGIN && prev_evt_ != JSON_ARRAY_BEGIN) { + os_ << '\n'; + } + os_ << *curr_indent_ << text; + return; + + case JSON_SYNTAX_OBJECT_COMMA: + case JSON_SYNTAX_ARRAY_COMMA: + os_ << text << '\n'; + return; + + default: + if (prev_evt_ != JSON_SYNTAX_OBJECT_COLON) { + os_ << *curr_indent_; + } + os_ << text; + } + } + + static void handle_event( + JSONParsingEventType evt, const JSONRawStringRef& text, void* user_data + ) { + BOOST_ASSERT(!text.empty()); + FormatHandler& self = *static_cast(user_data); + self(evt, text); + if (evt != JSON_SYNTAX_WHITESPACE) { + self.prev_evt_ = evt; + } + } + +private: + FormatHandler& operator=(const FormatHandler&); +}; + +} // namespace + +/*! + * \brief Formats (pretty-prints) a JSON document. + * + * All existing whitespace is ignored, replaced by new whitespace which indent + * each value based on its array/object nesting level. + * + * \param json_utf8_text UTF-8 encoded text of the JSON document to format. + * \param len the byte-length of the text in \a json_utf8_text. + * \param[out] os the output stream to write the whitespace-less equivalent document. + * \param indent (Optional; default to tab) the \em unit indent whitespace string. + * \param key_sep (Optional; default to colon-space) the object key-value separator. + * \return true if \a json_utf8_text was entirely consumed by this parser, + * with leading and trailing JSON whitespace ignored, and the text in + * between was a valid well-formed JSON document; false otherwise. + */ +bool json_format( + const char* json_utf8_text, size_t len, std::ostream& os, + const std::string& indent, const std::string& key_sep +) { + try { + FormatHandler handler(os, indent, key_sep); + return 0 == json_parse_syntax( + json_utf8_text, len, &FormatHandler::handle_event, &handler + ); + } catch (const std::runtime_error&) { + return false; + } +} + +size_t json_format_multi_part( + const char* json_utf8_text, size_t len, std::ostream& os, + const std::string& indent, const std::string& key_sep +) { + FormatHandler handler(os, indent, key_sep); + const char* end = json_utf8_text + len; + size_t remaining = len; + size_t part_count = 0; + do { + OES_ASSERT(remaining <= len, return part_count;); + const char* from = end - remaining; + remaining = json_parse_syntax( + from, remaining, &FormatHandler::handle_event, &handler + ); + os << std::endl; + ++part_count; + } while (remaining > 0); + return part_count; +} + +/*! + * \brief Formats (pretty-prints) a JSON document. + * + * All existing whitespace is ignored, replaced by new whitespace which indent + * each value based on its array/object nesting level. + * + * \param json_utf8_text UTF-8 encoded text of the JSON document to format. + * \param len the byte-length of the text in \a json_utf8_text. + * \param indent (Optional; default to tab) the \em unit indent whitespace string. + * \param key_sep (Optional; default to colon-space) the object key-value separator. + * \return the formatted document on success; an empty string otherwise. + */ +std::string json_format( + const char* json_utf8_text, size_t len, + const std::string& indent, const std::string& key_sep +) { + std::ostringstream oss; + return json_format(json_utf8_text, len, oss, indent, key_sep)? oss.str(): ""; +} + +/*************************************************************************/ + +/*! + * \brief Removes all insignificant whitespace from a JSON document. + * + * \param json_utf8_text UTF-8 encoded text of the JSON document. + * \param len the byte-length of the text in \a json_utf8_text. + * \param[out] os the output stream to write the whitespace-less equivalent document. + * \return true if \a json_utf8_text was entirely consumed by this parser, + * with leading and trailing JSON whitespace ignored, and the text in + * between was a valid well-formed JSON document; false otherwise. + * + * \note Resulting document is just one (possibly very long) line. + */ +bool json_remove_whitespace( + const char* json_utf8_text, size_t len, std::ostream& os +) { + try { + JSONSyntaxParser parser(print_no_whitespace_handler, &os); + return 0 == parser.parse(json_utf8_text, len); + } catch (const std::runtime_error&) { + return false; + } +} + +/*! + * \brief Removes all insignificant whitespace from a JSON document. + * + * \param json_utf8_text UTF-8 encoded text of the JSON document. + * \param len the byte-length of the text in \a json_utf8_text. + * \return the whitespace-less document on success; an empty string otherwise. + * + * \note Resulting document is just one (possibly very long) line. + */ +std::string json_remove_whitespace(const char* json_utf8_text, size_t len) { + std::ostringstream oss; + return json_remove_whitespace(json_utf8_text, len, oss)? oss.str(): ""; +} + +/*************************************************************************/ + +/*! + * \fn template std::string to_json_text(const T&) + * \brief Serializes a value to JSON text. + * + * Convenience function that expects a json_write(JSONWriter& json, const T& t) + * overload to exist, and is equivalent to the following code snippet: + * \code + * std::ostringstream oss; + * JSONWriter json(oss); + * json_write(json, t); + * return oss.str(); + * \endcode + * + * \tparam T the type to serialize to JSON thanks to json_write() overload + * \param t the value or object to serialize. + * \return the serialized JSON text. + * + * \note The json_write() overload for T must be either in the same namespace + * as T (to be found via Argument-Dependent-Lookup, also known as Koenig + * Lookup), or be declared in the oes::core namespace. + * + * \sa JSONWriter + * \sa json_parse(const char*, size_t, JSONHandler&) + */ + +/*! + * \fn template void json_write(JSONWriter&, const std::map&) + * \brief Writes a map of key/value pairs. + * + * Writes key/value pairs \p key_values to the JSON writer \p json as a JSON + * object. This function expects a json_write(JSONWriter& json, const V&) to + * write the map values. Overloads already exist for all JSON basic types. + * (See notes in to_json_text()). + * + * The resulting document can be later parsed with json_parse(). + * + * \tparam V type of the values in the map + * \param json the JSON writer to write to. + * \param key_values a map of key/value pairs + * + * \see json_parse(const char*, size_t, std::map&, bool) + */ + +/*! + * \fn template void json_write(JSONWriter&, const std::vector&) + * + * \tparam V type of the values in the vector + * \param json the JSON writer to write to + * \param values a vector of values + * \sa JSONWriter::values(const C&) + */ + +/*! + * \fn template void json_write(JSONWriter&, const std::set&) + * \tparam V type of the values in the set + * \param json the JSON writer to write to. + * \param values a set of values + * \sa JSONWriter::values(const C&) + */ + +/*! + * \fn template void json_write(JSONWriter& json, const boost::array& values) + * \tparam V type of the values in the array + * \param json the JSON writer to write to. + * \param values an array of values + * \sa JSONWriter::values(const C&) + */ + +/*************************************************************************/ + +} // namespace core +} // namespace oes diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.h new file mode 100644 index 0000000000000000000000000000000000000000..b3bce6064a0f5fdf89f83615d85dd46942c81a6d --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.h @@ -0,0 +1,440 @@ +// ============================================================================ +// Copyright 2013-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_TYPES_CORE_JSON_H +#define OES_CORE_TYPES_CORE_JSON_H + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace oes { +namespace core { + +/****************************************************************************/ + +struct OES_CORE_UTILS_EXPORT JSONWriterOptions { + // TODO: format option: indent, all-ascii, floating-point formatting, ... + + VersionNumber version_; + + bool omit_defaults_ = false; + + bool null_on_empty_ = false; +}; + +/****************************************************************************/ + +class OES_CORE_UTILS_EXPORT JSONWriter { +public: + explicit JSONWriter( + std::ostream& os, const JSONWriterOptions& options = JSONWriterOptions() + ); +#if (__cplusplus > 199711L) || (defined (_MSVC_LANG) && (_MSVC_LANG >= 201402L)) + ~JSONWriter() noexcept(false) ; // no meant to be extended/derived +#else + ~JSONWriter() ; // no meant to be extended/derived +#endif + + bool ok() const; + void reset(); + + const JSONWriterOptions& options() const; + + JSONWriter& number(int value); + JSONWriter& number(int64_t value); + JSONWriter& number(uint64_t value); + JSONWriter& number(unsigned int value); +#ifdef _WIN32 + JSONWriter& number(unsigned long value); +#endif + JSONWriter& number(float value); + JSONWriter& number(double value); + JSONWriter& string(const std::string& value, bool null_on_empty = false); + JSONWriter& json_bool(bool value); + JSONWriter& json_null(); + JSONWriter& json_true(); + JSONWriter& json_false(); + + JSONWriter& push_array(); + JSONWriter& pop_array(); + + JSONWriter& push_object(); + JSONWriter& key(const std::string& key); + JSONWriter& pop_object(); + + template JSONWriter& value(const V& value); // see ./json.hpp + template JSONWriter& values(const C& container); + template JSONWriter& size(const C& container); + JSONWriter& size(size_t value); + + JSONWriter& whitespace(const std::string& ws, bool before_comma_colon); + +private: + JSONWriter& operator=(const JSONWriter&); + JSONWriter(const JSONWriter&); + + void flush_pending_ws(); + void assert_expects_key(); + void assert_expects_value(); + template JSONWriter& push_value(T t); + +private: + std::ostream& os_; + std::vector end_; + std::vector pending_ws_; + JSONWriterOptions options_; +}; + +OES_FWD_DECL_SHARED_PTR(JSONWriter); + +inline JSONWriter& JSONWriter::size(size_t value) { + return number(value); +} + +/****************************************************************************/ + +class OES_CORE_UTILS_EXPORT JSONHandler { +public: + const bool return_code; + +public: + explicit JSONHandler(bool return_code = false); + virtual ~JSONHandler(); + + virtual bool handle_object_begin(); + virtual bool handle_object_key(const std::string& key); + virtual bool handle_object_comma(); + virtual bool handle_object_end(); + + virtual bool handle_array_begin(); + virtual bool handle_array_comma(); + virtual bool handle_array_end(); + + virtual bool handle_number(int); + virtual bool handle_number(int64_t); + virtual bool handle_number(uint64_t); + virtual bool handle_number(double value); + virtual bool handle_string(const std::string& value); + virtual bool handle_boolean(bool value); + virtual bool handle_null(); + + virtual bool handle_whitespace(const std::string& ws, bool before_comma_colon); + +private: // no assignment + JSONHandler& operator=(const JSONHandler&); +}; + +OES_FWD_DECL_SHARED_PTR(JSONHandler); + +OES_CORE_UTILS_EXPORT JSONHandler_ptr json_handler_writer(std::ostream& os); +OES_CORE_UTILS_EXPORT JSONHandler_ptr json_handler_writer(const JSONWriter_ptr& writer); + +/****************************************************************************/ + +struct OES_CORE_UTILS_EXPORT JSONToken { + const char* beg_; + const char* end_; + + JSONToken(const char* begin = 0, const char* end = 0); + + bool empty() const; + size_t size() const; + bool is_quoted() const; + const char* find(char c) const; + bool matches(const char* text, size_t len) const; + bool matches_unquote(const char* text, size_t len) const; + + std::string to_string(bool unquote = false) const; + std::ostream& print(std::ostream& os) const; +}; + +inline std::ostream& operator<<(std::ostream& os, const JSONToken& token) { + return token.print(os); +} + +/****************************************************************************/ + +// TODO: Merge with JSONToken, collapsing union into opaque uint flags_ +struct OES_CORE_UTILS_EXPORT JSONRawStringRef : public JSONToken { + union { + struct { + bool is_negative_; + bool has_dot_; + bool has_exp_; + } num_; + + struct { + bool not_ascii_; + bool has_escape_; + bool has_unicode_; + } str_; + + bool before_comma_colon_; + }; + + JSONRawStringRef(const char* begin = 0, const char* end = 0); + + JSONRawStringRef& reset(const char* begin); + JSONRawStringRef& endsAt(const char* end); + + void decode(std::string& buffer, bool unquote = false) const; +}; + +/****************************************************************************/ + +// TODO: Rename JSONTokenType +enum JSONParsingEventType { + //! Special end-of-document token. + JSON_END = 0, + + // Value tokens. + JSON_NULL, + JSON_TRUE, + JSON_FALSE, + JSON_STRING, + JSON_NUMBER, + + JSON_OBJECT_BEGIN, + JSON_OBJECT_KEY, + JSON_OBJECT_END, + + JSON_ARRAY_BEGIN, + JSON_ARRAY_END, + + // Syntax-only tokens (not emitted by JSONReader by default). + JSON_SYNTAX_WHITESPACE, + JSON_SYNTAX_OBJECT_COLON, + JSON_SYNTAX_OBJECT_COMMA, + JSON_SYNTAX_ARRAY_COMMA +}; + +typedef void (*JSONSyntaxHandler)( + JSONParsingEventType evt, const JSONRawStringRef& text, void* handler_data +); + +OES_CORE_UTILS_EXPORT size_t json_parse_syntax( + const char* json_utf8_text, size_t len, + JSONSyntaxHandler handler, void* handler_data +); + +/****************************************************************************/ + +struct OES_CORE_UTILS_EXPORT JSONParserOptions { + JSONParserOptions(); + + bool allow_scalar_; // TODO + bool syntax_mode_; // TODO + bool ignore_trailing_content_; + + // limits (to avoid Denial-Of-Service attacks) + size_t max_depth_; + size_t max_array_size_; + size_t max_object_size_; + size_t max_string_byte_size_; + size_t max_objkey_byte_size_; +}; + +/****************************************************************************/ + +struct JSONPullParser; + +class OES_CORE_UTILS_EXPORT JSONReader { +private: + struct SavePointImpl; + +public: + using SavePoint = std::shared_ptr; + using Scope = std::shared_ptr; + +public: + // TODO: Ctor taking JSONInput abstraction. + // TODO: Resumable reader on invalid input. + // TODO: Line:Column location of errors, and start of token. + JSONReader( + const char* json_utf8_text, size_t len, + const JSONParserOptions& options = JSONParserOptions() + ); + ~JSONReader(); + + JSONParsingEventType peek() const; + JSONParsingEventType next(); + JSONParsingEventType current(); + + Scope scope(); + + SavePoint savepoint(); + bool rollback_to(const SavePoint& savepoint); + + size_t skip_next(); + size_t skip_current(); + + JSONToken token(); + size_t depth(); + size_t count(); + + bool is_integral(); + + int get_int(); + int64_t get_int64_t(); + uint64_t get_uint64_t(); + + float get_float(); + double get_double(); + + std::string get_string(); + std::string get_string_or_null(); + + bool get_boolean(); + + std::string get_key(); + bool is_key(const char* key); + bool is_key(const char* key, size_t len); + + template bool parse_array(F lambda); // see ./json.hpp + template bool try_parse_array(F lambda); + + template bool try_parse(V& value); + template bool try_parse_member(V& value); + + static bool test_token_type(JSONParsingEventType expected, JSONParsingEventType received); + static bool test_token_string(std::string expected, std::string received); + +private: // no copy/assignment + JSONReader& operator=(const JSONReader&) = delete; + JSONReader(const JSONReader&) = delete; + + JSONReader(const std::shared_ptr& outer_parser); + +private: + std::shared_ptr impl_; +}; + +/****************************************************************************/ + +// TODO: Rename JSONValueType +enum JSONType { + JSON_TYPE_INVALID, + JSON_TYPE_OBJECT, + JSON_TYPE_ARRAY, + JSON_TYPE_STRING, + JSON_TYPE_NUMBER, + JSON_TYPE_BOOLEAN, + JSON_TYPE_ANY +}; + +OES_CORE_UTILS_EXPORT JSONType json_type_of(const std::string& json_expr); + +OES_CORE_UTILS_EXPORT std::string json_type_string_of(JSONType type_value); + +OES_CORE_UTILS_EXPORT JSONType json_type_value_of(const std::string& type_string); + +/****************************************************************************/ + +OES_CORE_UTILS_EXPORT bool json_parse( + const char* json_utf8_text, size_t len, JSONHandler& handler +); + +OES_CORE_UTILS_EXPORT bool json_parse( + const char* json_utf8_text, size_t len, + std::map& key_values, + bool throw_on_duplicates = false, + JSONType value_type = JSON_TYPE_ANY +); + +OES_CORE_UTILS_EXPORT bool json_parse( + const char* json_utf8_text, size_t len, + std::vector& values, + JSONType value_type = JSON_TYPE_ANY +); + +OES_CORE_UTILS_EXPORT bool json_validate_syntax( + const char* json_utf8_text, size_t len +); + +OES_CORE_UTILS_EXPORT bool json_format( + const char* json_utf8_text, size_t len, std::ostream& os, + const std::string& indent = "\t", const std::string& key_sep = ": " +); +OES_CORE_UTILS_EXPORT std::string json_format( + const char* json_utf8_text, size_t len, + const std::string& indent = "\t", const std::string& key_sep = ": " +); +OES_CORE_UTILS_EXPORT size_t json_format_multi_part( + const char* json_utf8_text, size_t len, std::ostream& os, + const std::string& indent = "\t", const std::string& key_sep = ": " +); + +OES_CORE_UTILS_EXPORT bool json_remove_whitespace( + const char* json_utf8_text, size_t len, std::ostream& os +); +OES_CORE_UTILS_EXPORT std::string json_remove_whitespace( + const char* json_utf8_text, size_t len +); + +/****************************************************************************/ + +inline void json_write(JSONWriter& json, bool value) { + json.json_bool(value); +} +inline void json_write(JSONWriter& json, int value) { + json.number(value); +} +inline void json_write(JSONWriter& json, uint64_t value) { + json.number(value); +} +inline void json_write(JSONWriter& json, int64_t value) { + json.number(value); +} +inline void json_write(JSONWriter& json, unsigned int value) { + json.number(value); +} +#ifdef _WIN32 +inline void json_write(JSONWriter& json, unsigned long value) { + json.number(value); +} +#endif +inline void json_write(JSONWriter& json, float value) { + json.number(value); +} +inline void json_write(JSONWriter& json, double value) { + json.number(value); +} +inline void json_write(JSONWriter& json, const char* value) { + if (value == nullptr) { + json.json_null(); + } else { + json.string(value); + } +} +inline void json_write(JSONWriter& json, const std::string& value) { + json.string(value); +} + +/****************************************************************************/ + +} // namespace core +} // namespace oes + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.hpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.hpp new file mode 100644 index 0000000000000000000000000000000000000000..587b65872c82d22b26ab34b425a3de567dfc283f --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Json.hpp @@ -0,0 +1,233 @@ +// ============================================================================ +// Copyright 2013-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_JSON_HPP +#define OES_CORE_UTILS_JSON_HPP + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace oes { +namespace core { + +/****************************************************************************/ + +template +std::string to_json_text(const V& value) { + std::ostringstream oss; + JSONWriter json(oss); + // json_write() overload must be defined in the oes::core namespace, + // or defined in the same namespace as T (to be found via ADL). + json_write(json, value); + return oss.str(); +} + +template +void json_write_value_helper(JSONWriter& json, const V& value) { + // json_write() overload must be defined in the oes::core namespace, + // or defined in the same namespace as T (to be found via ADL). + json_write(json, value); +} + +/****************************************************************************/ + +template +JSONWriter& JSONWriter::value(const V& value) { + // name lookup somehow fails if I call json_write(*this, value); directly. + oes::core::json_write_value_helper(*this, value); + return *this; +} + +template +JSONWriter& JSONWriter::size(const C& container) { + return number(container.size()); +} + +template +JSONWriter& JSONWriter::values(const C& container) { + push_array(); + for (const auto& elem : container) { + json_write(*this, elem); + } + pop_array(); + return *this; +} + +/****************************************************************************/ + +template +void json_write(JSONWriter& json, const std::map& key_values) { + json.push_object(); + for (const auto& entry : key_values) { + json.key(entry.first); + json_write(json, entry.second); + } + json.pop_object(); +} + +template +void json_write(JSONWriter& json, const std::vector& values) { + json.values(values); +} + +template +inline void json_write(JSONWriter& json, const std::set& values) { + json.values(values); +} + +template +void json_write(JSONWriter& json, const boost::array& values) { + json.values(values); +} + +/*****************************************************************************/ + +template +void json_write_kv_unless_empty( + JSONWriter& json, const std::string& key, const T& value +) { + if (!json.options().omit_defaults_ || !value.empty()) { + json.key(key).value(value); + } +} + +template +void json_write_kv_unless_equal( + JSONWriter& json, const std::string& key, T value, T ndv = T() +) { + if (!json.options().omit_defaults_ || value != ndv) { + json.key(key).value(value); + } +} + +/****************************************************************************/ + +template +bool JSONReader::parse_array(F lambda) { + if (next() != JSON_ARRAY_BEGIN) { + return false; + } + const size_t my_depth = depth(); + while (peek() != JSON_ARRAY_END && depth() == my_depth) { + if (!lambda()) { + // whether array-end or real failure + // determined by last return statement. + break; + } + if (depth() != my_depth) { + return false; // buggy lambda + } + } + return next() == JSON_ARRAY_END && (depth() + 1) == my_depth; +} + +template +bool JSONReader::try_parse_array(F lambda) { + try { + return parse_array(lambda); + } catch (const std::runtime_error& /*ex*/) { + // TODO: log error + return false; + } +} + +template +bool json_parse_array(JSONReader& json, C& container) { + using value_t = typename C::value_type; + return json.try_parse_array([&]() { + value_t val; + if (!value_t::parse(json, val)) { + return false; + } + container.emplace_back(std::move(val)); + return true; + }); +} + +template +bool JSONReader::try_parse(V& value) { + try { + return json_parse(*this, value); + } catch (const std::runtime_error& /*ex*/) { + // TODO: log error + return false; + } +} + +template +bool JSONReader::try_parse_member(V& value) { + try { + return V::parse(*this, value); + } catch (const std::runtime_error& /*ex*/) { + // TODO: log error + return false; + } +} + +/****************************************************************************/ + +template +bool json_parse_object( + JSONReader& json, size_t min_keys, size_t max_keys, F&& lambda +) { + if (json.next() != JSON_OBJECT_BEGIN) { + return false; + } + if (json.peek() == JSON_OBJECT_END) { + json.next(); + return min_keys == 0; // true when empty object OK; false otherwise. + } + + const size_t inner_depth = json.depth(); + + size_t i = 0; + for (; (json.next() == JSON_OBJECT_KEY) && + (json.depth() == inner_depth) && + (i < max_keys); ++i + ) { + if (!lambda()) { + return false; + } + // catch buggy lambda. Not foul proof, could be consuming + // several key-value pairs for example. Better than nothing. + // Foolproof would require a json.savepoint() around reading + // the value, which is not free, and also implies passing the + // key to the lamdba explicitly. + if (json.depth() != inner_depth) { + return false; + } + } + + return + (min_keys <= i && i <= max_keys) && + (json.current() == JSON_OBJECT_END) && + (json.depth() == (inner_depth - 1)) + ; +} + +/****************************************************************************/ + +} // namespace core +} // namespace oes + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Logger.cpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Logger.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7cbbe67dc1c5b53664f5ecdb1b2c24e46c80f54 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Logger.cpp @@ -0,0 +1,1443 @@ +// ============================================================================ +// Copyright 2008-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + + +/*! \file oes/core/utils/Logger.h + * \brief Message logging framework + */ + + +/* TRANSLATOR Format */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#define VC_EXTRALEAN +#define WIN32_LEAN_AND_MEAN +#include +#endif +#include + +namespace bpo = boost::program_options; + +/***************************************************************************/ + +namespace oes { + namespace core { + + namespace { + + /*! + * \brief Forwards the log messages to the console + * \details UI messages are discarded + */ + struct ConsoleMessageHandler : MessageHandler { + ConsoleMessageHandler() : MessageHandler(false), formatter_() { + //The Console only cares for log messages: discard UI message + connectFilter(Logger::createSeverityFilter(Message::first_log_severity, Message::last_log_severity)); + } + virtual void print(const String& txt) { + std::cout << txt; + } + virtual void handleMessage(const Message_ptr& message) override { + const String txt = formatter_ ? formatter_->format(message) : defaultFormatter_.format(message); + print(txt); + } + virtual bool setFormatter(ConstMessageFormatter_ptr formatter) { + formatter_ = formatter; + return true; + } + static const TextMessageFormatter defaultFormatter_; + ConstMessageFormatter_ptr formatter_; + }; + const TextMessageFormatter ConsoleMessageHandler::defaultFormatter_; + + struct DebugConsoleMessageHandler : ConsoleMessageHandler { + virtual void print(const String& txt) { +#if defined(_WIN32) + if( !txt.empty() ) { + const String::size_type i_last = txt.size () - 1; + + //Enforce the proper formatting: each messages shall appear in the separate line. + //Those messages that have a trailing '\n' (patterned as "...\n" or "...\0\n") are + //passed as they are, '\n' is appended to the others. + if( '\n' != txt [i_last] && !('\0' == txt [i_last] && '\n' == txt [i_last - 1]) ) { + //The message does not have a leading '\n', so append a one. + String formatted_txt = txt; + formatted_txt.push_back('\n'); + OutputDebugStringA(formatted_txt.c_str()); + } else { + OutputDebugStringA(txt.c_str()); + } + } +#else + OES_UNUSED(txt); +#endif + } + }; + + struct BySeverityMessageFilter : MessageFilter { + BySeverityMessageFilter( + Message::Severity mins, Message::Severity maxs + ) : MessageFilter() + , minimumSeverity_(mins) + , maximumSeverity_(maxs) + { + } + + virtual bool filter(Message_ptr message) const { + const auto s = message->severity(); + if (minimumSeverity_ <= s && s <= maximumSeverity_) { + return false; // Not filtered + } + return true; // Filtered + } + + const Message::Severity minimumSeverity_; + const Message::Severity maximumSeverity_; + }; + + } // namespace + + /*! \class Message oes/core/utils/Logger.h + * \brief Base class of messages. + * \ingroup oes_core_utils_logging + */ + + /*! + * \brief Base message constructor + */ + Message::Message(const Format& format) : + time_(boost::posix_time::second_clock::local_time()), + lineNumber_(0), + depth_(0), + format_(format) + { + } + + /*! + * \brief Destroys a message. + */ + Message::~Message() { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity Message::severity() const { + return unknown; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& Message::prefix() const { + static String s; + return s; + } + + /*! + * Returns a string representation of the current time + */ + String Message::time_string() const { + return boost::posix_time::to_simple_string(time_); + } + + /*! + * Returns the formatted message with the prefix. + */ + Format Message::format() const { + return format_; + } + + /*! + * Sets the formatted text of the message. + * \param[in] format the formatted text of the message + */ + void Message::setFormat(const Format& format) { + format_ = format; + } + + /*! + * \fn std::ostream& operator << (std::ostream& os, const Message& message); + * Prints message \a message to output stream \a os. + * Printing is done using a StdFormatter + */ + + /*! \brief Converts an input Message::Severity to its string representation + * \param[in] severity The severity to be converted to string + * \return The string representation of a severity, lower-case + */ + const String Message::severityToString(Message::Severity severity) { + switch (severity) { + case Message::debug: return "debug"; + case Message::verbose: return "verbose"; + case Message::info: return "info"; + case Message::warning: return "warning"; + case Message::error: return "error"; + case Message::alert: return "alert"; + case Message::critical: return "critical"; + case Message::fatal: return "fatal"; + case Message::status: return "status"; + case Message::notice: return "notice"; + default: return "unknown"; + }; + } + + /*! \brief Converts an input String to its Message::Severity equivalent + * \details + * - Returns Message::unknown if the input string does not represent + * any severity level (e.g. "not_a_severtity" returns Message::unknown) + * - The input \p severity is processed in a case-insensitive manner + * (e.g. "info" and "INFO" both returns Message::info). + * \param[in] severity An input string representing a message severity + * (among status|debug|verbose|info|notice|warning|error|alert|critical|fatal). + */ + Message::Severity Message::severityFromString(const String& severity) { + typedef std::map SeverityNameMap; + static const SeverityNameMap severityNames{ + { "unknown", Message::unknown }, + { "debug", Message::debug }, + { "verbose", Message::verbose }, + { "info", Message::info }, + { "warning", Message::warning }, + { "error", Message::error }, + { "alert", Message::alert }, + { "critical", Message::critical }, + { "fatal", Message::fatal }, + { "status", Message::status }, + { "notice", Message::notice } + }; + SeverityNameMap::const_iterator itr = severityNames.find( + boost::algorithm::to_lower_copy(severity) + ); + if( itr == severityNames.end() ) { + return Message::unknown; + } + return itr->second; + } + + /***************************************************************************/ + + /*! \class FatalMessage oes/core/utils/Logger.h + * \brief Format for fatal error messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted fatal error message. + */ + FatalMessage::FatalMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity FatalMessage::severity() const { + return fatal; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& FatalMessage::prefix() const { + static String prefix("Fatal"); + return prefix; + } + + /***************************************************************************/ + + /*! \class CriticalMessage oes/core/utils/Logger.h + * \brief Format for error messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted critical error message. + */ + CriticalMessage::CriticalMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity CriticalMessage::severity() const { + return critical; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& CriticalMessage::prefix() const { + static String prefix("Critical"); + return prefix; + } + + /***************************************************************************/ + + /*! \class AlertMessage oes/core/utils/Logger.h + * \brief Format for error messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted alert message. + */ + AlertMessage::AlertMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity AlertMessage::severity() const { + return alert; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& AlertMessage::prefix() const { + static String prefix("Alert"); + return prefix; + } + + /***************************************************************************/ + + /*! \class ErrorMessage oes/core/utils/Logger.h + * \brief Format for error messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted error message. + */ + ErrorMessage::ErrorMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity ErrorMessage::severity() const { + return error; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& ErrorMessage::prefix() const { + static String prefix("Error"); + return prefix; + } + + /***************************************************************************/ + + /*! \class WarningMessage oes/core/utils/Logger.h + * \brief Format for warning messages. + * \ingroup oes_core_utils_logging + */ + + /*! + * Creates a formatted warning message. + */ + WarningMessage::WarningMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity WarningMessage::severity() const { + return warning; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& WarningMessage::prefix() const { + static String prefix("Warning"); + return prefix; + } + + /***************************************************************************/ + + /*! \class NoticeMessage oes/core/utils/Logger.h + * \brief Format for notice messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted notice message. + */ + NoticeMessage::NoticeMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity NoticeMessage::severity() const { + return notice; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& NoticeMessage::prefix() const { + static String prefix("Notice"); + return prefix; + } + + /***************************************************************************/ + + /*! \class CursorNoticeMessage oes/core/utils/Logger.h + * \brief Format for notice messages appearing next to the cursor + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted notice message. + */ + CursorNoticeMessage::CursorNoticeMessage(const Format& format): + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity CursorNoticeMessage::severity() const { + return cursor_notice; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& CursorNoticeMessage::prefix() const { + static String prefix("Notice"); + return prefix; + } + /***************************************************************************/ + + /*! \class InfoMessage oes/core/utils/Logger.h + * \brief Format for Info messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted info message. + */ + InfoMessage::InfoMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity InfoMessage::severity() const { + return info; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& InfoMessage::prefix() const { + static String prefix("Info"); + return prefix; + } + + /***************************************************************************/ + + /*! \class VerboseMessage oes/core/utils/Logger.h + * \brief Format for Verbose messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted verbose message. + */ + VerboseMessage::VerboseMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity VerboseMessage::severity() const { + return verbose; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& VerboseMessage::prefix() const { + static String prefix("Verbose"); + return prefix; + } + + /***************************************************************************/ + + /*! \class DebugMessage oes/core/utils/Logger.h + * \brief Format for debug messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted debug message. + */ + DebugMessage::DebugMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity DebugMessage::severity() const { + return debug; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& DebugMessage::prefix() const { + static String prefix("Debug"); + return prefix; + } + + /***************************************************************************/ + + /*! \class StatusMessage oes/core/utils/Logger.h + * \brief Format for Status messages. + * \ingroup oes_core_utils_logging + */ + + + /*! + * Creates a formatted Logging message. + */ + StatusMessage::StatusMessage(const Format& format) : + Message(format) { + } + + /*! + * Returns the severity for this type of message. + */ + Message::Severity StatusMessage::severity() const { + return status; + } + + /*! + * Returns the string that prefix this type of message. + */ + const String& StatusMessage::prefix() const { + static String prefix("Status"); + return prefix; + } + + /****************************************************************************/ + + /*! \class NotificationMessagesBufferEvent oes/core/utils/Logger.h + * \brief Buffer event for notice messages. + * + * Sent to turn on or off the buffering for messages sent to the notification widget. + * + * When buffering is turned on, the following notification messages are appended to the end + * of the existing notice buffer and not displayed. + * and displayed as a whole in the notification widget. + * + * When buffering is turned off, the next notice message sent will clear + * all the previously buffered messages. Note that, turning off buffering + * does not clear the current notice messages (i.e. the buffer). + * Clearing of the buffer occurs when the next, non-buffered message is sent. + * + * \remarks The preferred way of utilizing buffering for notice messages + * is via the NotificationMessagesScopedBuffering class. Unless absolutely + * necessary, developers are discouraged to send PrintPrettyBufferEvents + * directly. + * + * \see NotificationMessagesScopedBuffering + * + * \ingroup oes_core_utils_logging + */ + + /*! + * Constructs a notice buffer event that will turn the buffering on. + * \param[in] on Only used to distinguish the two constructors. + * Has no effect if the buffering was already on. + */ + NotificationMessagesBufferEvent::NotificationMessagesBufferEvent(const turn_buffering_on_tag& on) : + Message(Format("")), + turn_buffering_on_(true) { + OES_UNUSED(on); + } + + /*! + * Constructs a notice buffer event that will turn the buffering on. + * \param[in] off Only used to distinguish the two constructors. + * Has no effect if the buffering was already off. + */ + NotificationMessagesBufferEvent::NotificationMessagesBufferEvent(const turn_buffering_off_tag& off) : + Message(Format("")), + turn_buffering_on_(false) { + OES_UNUSED(off); + } + + /*! + * Returns true if this NoticeBufferEvent is to turn on the buffering of + * the notice messages when received by the appropriate event consumer. + * If false, the event is to turn off the buffering. + */ + bool NotificationMessagesBufferEvent::turn_buffering_on() const { + return turn_buffering_on_; + } + + /****************************************************************************/ + + /*! \class NotificationMessagesScopedBuffering oes/core/utils/Logger.h + * \ingroup oes_core_utils_logging + * \brief Convenience class to turn on and off print pretty buffering. + * + * NotificationMessagesScopedBuffering turns on buffering for notification messages upon + * construction and turns off buffering upon destruction by sending + * the necessary NotificationMessagesBufferEvent to the appropriate event consumer. + * + * In other words, this RAII-based convenience class ties the lifetime of + * print pretty buffering to its own lifetime. When an instance of this class + * goes out of scope and thus destructed, print pretty buffering is + * automatically turned off. + * + * \remarks The preferred way of utilizing buffering for notification messages + * is via the NotificationMessagesScopedBuffering class. NotificationMessagesBufferEvent + * should not be sent directly. + * + * \see NotificationMessagesBufferEvent + */ + + + /*! + * Default constructor. + * Turns on buffering by sending a NotificationMessagesBufferEvent. + */ + NotificationMessagesScopedBuffering::NotificationMessagesScopedBuffering() { + Message_ptr message( + new NotificationMessagesBufferEvent( NotificationMessagesBufferEvent::turn_buffering_on_tag() ) + ); + for (const MessageHandler_ptr& handler : Logger::instance().handlers_) { + handler->receive(message); + } + } + + /*! + * Destructor. + * Turns off buffering by sending a NotificationMessagesBufferEvent. + */ + NotificationMessagesScopedBuffering::~NotificationMessagesScopedBuffering() { + Message_ptr message( + new NotificationMessagesBufferEvent(NotificationMessagesBufferEvent::turn_buffering_off_tag()) + ); + for (const MessageHandler_ptr& handler : Logger::instance().handlers_) { + handler->receive(message); + } + } + + /****************************************************************************/ + + /*! \class MessageHandler oes/core/utils/Logger.h + * \ingroup oes_core_utils_logging + * + * \brief MessageHandler provides a mean for redirecting messages sent + * to Logger streams + * \details (eg: update a widget appearance, write + * to * a pipe, ...). + * + * Users who want to capture messages and redirect them at their + * convenience should create a derived class and redefine function + * receive(). + */ + + /*! \brief Base class constructor + * \param useSeverityFilter true to apply the severity filter from the Logger. This + * parameter is true by default. + */ + MessageHandler::MessageHandler(bool useSeverityFilter) { + if (useSeverityFilter) { + if (const auto& filter = oes::core::Logger::instance().severityFilter()) { + connectFilter(filter); + } + } + } + + /*! \brief Destroys the message handler */ + MessageHandler::~MessageHandler() { + } + + /*! + * \brief Processes a message + * \details + * This function is called when a new message has arrived on the stream + * The handler is connected to(see Logger::connectHandler). + */ + void MessageHandler::receive(const Message_ptr& message) { + if (isFiltered(message)) { + return; + } + handleMessage(message); + } + + /*! + * \fn MessageHandler::handleMessage(const Message_ptr&) + * \brief Called to process the message if it is not filtered + */ + + /*! \brief Installs a new message filter message filtering. + * \param[in] filter The new filter to install + */ + void MessageHandler::connectFilter(const ConstMessageFilter_ptr& filter) { + OES_ASSERT(filter, return;); + filters_.push_back(filter); + } + + /*! \brief Removes a message filter + * \param[in] filter The filter to be removed (may not be null), the filter is not deleted + * \see connectFilter(ConstMessageFilter_ptr) + * \return true if the filter was removed + */ + bool MessageHandler::disconnectFilter(const ConstMessageFilter_ptr& filter) { + OES_ASSERT(filter, return false;); + if (filters_.empty()) { + return false; + } + auto itr = std::remove(filters_.begin(), filters_.end(), filter); + if (itr == filters_.end()) { + return false; + } + filters_.erase(itr, filters_.end()); + return true; + } + + /*! \brief Removes all filters + * \see connectFilter(ConstMessageFilter_ptr) + */ + void MessageHandler::disconnectFilters() { + filters_.clear(); + } + + /*! + * \return true if the message is filtered + * \details Alls filters are given the opportunity to filter + */ + bool MessageHandler::isFiltered(const Message_ptr& message) { + bool discarded = false; + for (const ConstMessageFilter_ptr& filter : filters_) { + // filter cannot be null. See assert in connectFilter() + if (filter->filter(message)) { + discarded = true; + } + } + return discarded; + } + + /*! \brief Assigns a new message formatter + * message formatting. + * \details + * - Default implementation does not support message formatting, it is + * up to concrete classes to accept and make use of the new formatter. + * \param[in] formatter The new formatter, may be null to remove formatting + * \return true if the handler supports message formatting and accepts the + * new formatter, false otherwise. + */ + bool MessageHandler::setFormatter(ConstMessageFormatter_ptr formatter) { + return !formatter; + } + + /****************************************************************************/ + + /*! \class MessageFilter oes/core/utils/Logger.h + * \ingroup oes_core_utils_logging + * \brief Base class for message filters. + */ + + /*! \fn virtual bool MessageFilter::filter(Message_ptr message) const + * \brief Performs message filtering + * \details + * Note that the filter may choose to alter message itself if + * applicable, message is not provided as a const object. + * \retval true If the message is being filtered, the message should NOT be + * forwarded to handlers (case of global filters) or should NOT be printed + * (case of local filters). + * \retval false If the message is not filtered, the message should be + * forwarded to handlers (case of global filters) or should be printed + * (case of local filters). + * \param[in] message the message sent to the Logger + */ + + MessageFilter::~MessageFilter() { + } + + /****************************************************************************/ + + /*! \class MessageContext oes/core/utils/Logger.h + * \ingroup oes_core_utils_logging + * \brief Base class for message contexts. + * \details + * Message contexts allow to add contextual information to a message + * before it is sent to the message handlers. For example, + * MessageContext%s are useful to add information about the included + * file stack in case of parsing errors. This information is generally + * not available to the code that sends the error, but the parser can + * install a MessageContext in the Logger that provides this + * information. + */ + + /*! \fn virtual void MessageContext::apply(Message_ptr message) const + * \brief Applies a context to a message + * \details + * This function must replace the message formatted text by adding + * information to the message. + * \param[in] message the message sent to the Logger + * \return a new Message. + * \see Message::setFormat() + */ + + MessageContext::~MessageContext() { + } + + /****************************************************************************/ + + /*! \class MessageFormatter oes/core/utils/Logger.h + * \ingroup oes_core_utils_logging + * \brief Base class for message formatters. + */ + + /*! \fn MessageHandler::MessageHandler() + * Base class constructor. + */ + + /*! + * Destroys the message formatter + */ + MessageFormatter::~MessageFormatter() { + } + + /*! \fn MessageFormatter::format(Message_ptr message) const + * \brief Formats the string to the desired format and returns the + * resulting string + */ + + /*! \class XmlMessageFormatter oes/core/utils/Logger.h + * \ingroup oes_core_utils_logging + * \brief Formats messages to XML format + */ + + /*! + * \brief Format the \a message using XML format. + */ + String XmlMessageFormatter::format(Message_ptr message) const { + const Message& m = *message; + + std::stringstream s; + s << "<" << m.prefix() << ">\n"; + s << " \n"; + s << " " << m.function() << "\n"; + s << " " << m.lineNumber() << "\n"; + s << " " << m.function() << "\n"; + s << " " << m.depth() << "\n"; + s << " " << m.format() << "\n"; + s << "\n"; + + // Another alternative could be: + // s << "\n"; + // ... + // s << "\n"; + + return s.str(); + } + + /*! \class TextMessageFormatter oes/core/utils/Logger.h + * \ingroup oes_core_utils_logging + * \brief Formats messages to Text format + */ + + /*! + * \brief Creates a new text formatter. + * + * The messages passed to the formatter are formatted using the + * format specification \a format. + * + * Ordinary characters placed in the format string are copied without + * conversion. Conversion specifications are introduced by a '%' + * character, and terminated by a conversion specifier character, and + * are replaced in s as follows: + * + * - %P: message prefix (Fatal, Critical, ...) + * - %T: the current date and time in the form YYY-MM-DD HH:MM:SS + * - %F: The file file that sent the message + * - %L: The line number in the file + * - %f: The function in the file (if available) + * - %M: The message text + * - %%: expands to the single character '%' + * - any other characters are printed as is. + * + * If \a format is empty, the default format is taken instead: [%T] %P: %M + */ + TextMessageFormatter::TextMessageFormatter(const String& format) : + format_(format) + { + if( format_.empty() ) { + // Reset to the default format: + format_ = "[%T] %P: %M"; + } + } + + /*! + * \brief Formats a message to text format + * \details + * Formats the \a message using Text format according to the format + * specification passed in the class constructor. + */ + String TextMessageFormatter::format(Message_ptr message) const { + std::stringstream s; + bool gotPercent = false; + + for( String::const_iterator p = format_.begin(); p != format_.end(); ++p ) { + if( gotPercent ) { + gotPercent = false; + switch(*p) { + case '%': + s << '%'; + break; + case 'T': + s << message->time_string(); + break; + case 'P': + s << message->prefix(); + break; + case 'F': + s << message->file(); + break; + case 'L': + s << message->lineNumber(); + break; + case 'f': + s << message->function(); + break; + case 'D': + s << message->depth(); + break; + case 'M': + s << message->format(); + break; + default: + s << '%' << *p; + break; + } + } else { + if( *p == '%' ) { + gotPercent = true; + continue; + } else { + s << *p; + } + } + } + + return s.str(); + } + + /****************************************************************************/ + + /*! \class Logger oes/core/utils/Logger.h + * \ingroup oes_core_utils oes_core_utils_logging + * \brief Prints a message to a message stream. + * \details + * - By default, the logger comes with a console handler (sending messages + * to standard output) and a debug console handler (sending messages to + * the debugging console) in windows debug builds. + * - To customize this behavior, users can attach one or more message handlers to + * the message stream: handlers will receive the message in the order of their + * registration. + */ + + /*!\brief logger message send observer + * \details + * Enables to notify the PG Logger + */ + Logger::MessageSendObserver Logger::messageSendObserver_; + + /*!\class Logger::ProgramOptions oes/core/utils/Logger.h + * \brief Defines the oes::core::Logger program options. + * \details The program options are loaded by oes::core::Configurator objects + * are used for the oes::core::Logger configuration. The program options names defined here + * may be shared by master logger implementation. Currently it is shared by the PGLog, but can possibly + * be shared by some other one. + */ + + /*! + * \brief The name of the command line argument used for setting the logging level. + * \details This option name is exported and may be shared by the master loggers options. + */ + const char* Logger::ProgramOptions::logSeverity_ = "log_level"; + + /*! + * \brief The shared implementation of the boost program_options custom validation + * for program option of the oes::core::Message::Severity type. + * + * \param values + * \return the valid severity + * + * \exception boost::program_options::validation_error if the token does not correspondents + * to the valid value. + */ + Message::Severity Logger::ProgramOptions::validateSeverity(const std::vector& values) { + const std::string& severity_name = boost::program_options::validators::get_single_string(values); + + // The severity value token is case insensitive, + // the severity name expected by the Message::severityFromString is case insensitive. + const Message::Severity severity = Message::severityFromString(severity_name); + if (Message::unknown == severity) { + //Unexpected severity value + throw bpo::validation_error( + bpo::validation_error::invalid_option_value, "unexpected severity token - ", severity_name + ); + } + + return severity; + } + + /*! + * \brief the boost::program_options overloaded validate function for Message::Severity enum. + * + * This function is called by the boost infrastructure and converts the string token + * that correspondents with the log severity option to the valid Message::Severity value. + * + * \exception boost::program_options::validation_error if the token does not correspondents + * to the valid value. The exception is handled by the oes::core::Configurator infrastructure. + */ + void validate(boost::any& v, const std::vector& values, Message::Severity*, int) { + bpo::validators::check_first_occurrence(v); + const Message::Severity severity = Logger::ProgramOptions::validateSeverity(values); + v = boost::any(severity); + } + + Logger::Logger() : + filters_(), + contexts_(), + handlers_(), + pgLoggerFormatter_ ("%M") + { + connectHandler(createConsoleHandler()); +#if defined(OES_DEBUG_BUILD) && defined(_WIN32) + connectHandler(createDebugConsoleHandler()); +#endif + } + + Logger::~Logger() { + } + + /*! \brief Performs a basic configuration of the logger from command line arguments + * \details Consolidates some basic configuration of the logger from command line + * \par Recognized logging options: + * -log_level, installing a global message filter with the said level. The default + * value is 'debug' in debug builds and 'info' in release builds. + * help option is not supported, as the help shall be supported on the application level. + * \retval true if the logger is successfully configured. + * \retval false if configure process had failed. + */ + bool Logger::configure(int argc, char *argv[]) { + bool success = false; + + + // configure the message filter with the -log_level arguments + { +#ifdef OES_DEBUG_BUILD + const Message::Severity severity_default = Message::debug; +#else + const Message::Severity severity_default = Message::verbose; +#endif + bpo::options_description options("Logging options"); + options.add_options() + (ProgramOptions::logSeverity_, + bpo::value()->default_value(severity_default), + "Logging level (debug|verbose|info|warning|error|alert|critical|fatal|status|notice)" + ); + + Configurator logger_configurator(argc, argv); + Configurator::LoadResult_t load_result = logger_configurator.loadConfiguration(options); + + if (load_result) { + const boost::program_options::variables_map& logger_config_params = *load_result; + Message::Severity severity_from_option = + logger_config_params[ProgramOptions::logSeverity_].as(); + setSeverityFilter(severity_from_option); + success = true; + } + } + + // configure the console message filter : also follows the -log_level argument + // but with a different default value than the global filter + if (consoleHandler()) { + // by default filter debug, info messages on the console + const Message::Severity severity_default = Message::warning; + bpo::options_description options("Logging options"); + options.add_options() + (ProgramOptions::logSeverity_, + bpo::value()->default_value(severity_default), + "Logging level (debug|verbose|info|warning|error|alert|critical|fatal|status|notice)" + ); + + Configurator logger_configurator(argc, argv); + Configurator::LoadResult_t load_result = logger_configurator.loadConfiguration(options); + + if (load_result) { + const boost::program_options::variables_map& logger_config_params = *load_result; + Message::Severity severity_from_option = + logger_config_params[ProgramOptions::logSeverity_].as(); + consoleHandler()->connectFilter(Logger::createSeverityFilter(severity_from_option, Message::last_log_severity)); + } + } + + return success; + } + + /*! + * \brief Updates or sets the minimum severity filter. + * + * \param minimumSeverity the minimum logging level a message must have (inclusive) + * \param maximumSeverity (Optional; defaults to max_severity, use last_log_severity to filter + * the messages specific to the UI) the maximum logging + * level a message may have (inclusive). + * + * \details By default, the severity filter is driven by command line + * option with -log_level. + * This filter is NOT global. However, by default it is placed on all + * MessageHandlers unless requested otherwise in the handler constructor. If a severity filter + * has been placed on a handler it will be replaced. + */ + void Logger::setSeverityFilter( + Message::Severity minimumSeverity, + Message::Severity maximumSeverity + ) { + ConstMessageFilter_ptr newFilter = createSeverityFilter(minimumSeverity, maximumSeverity); + if (severity_filter_) { + // if it has been placed on existing message handlers, replace it + for (const MessageHandler_ptr& handler : handlers_) { + if (handler->disconnectFilter(severity_filter_)) { + handler->connectFilter(newFilter); + if (severity_filter_ == nullptr) { + break; + } + } + } + } + severity_filter_ = newFilter; + } + + /*! + * \return the severity filter (controlled by -log_level) if any + */ + ConstMessageFilter_ptr Logger::severityFilter() const { + return severity_filter_; + } + + /*! + * \brief Gets the current minimum logging level of the severity filter. + * + * If no severity filter is installed, returns Message::unknown. + * \sa setSeverityFilter + */ + Message::Severity Logger::minSeverityFilter() const { + if (severity_filter_) { + auto severity_filter = + boost::dynamic_pointer_cast(severity_filter_); + OES_ASSERT(severity_filter, return Message::unknown;); + return severity_filter->minimumSeverity_; + } + return Message::unknown; + } + + /*! + * \brief Gets the current maximum logging level of the severity filter. + * + * If no severity filter is installed, returns Message::unknown. + * \sa setSeverityFilter + */ + Message::Severity Logger::maxSeverityFilter() const { + if (severity_filter_) { + auto severity_filter = + boost::dynamic_pointer_cast(severity_filter_); + OES_ASSERT(severity_filter, return Message::unknown;); + return severity_filter->maximumSeverity_; + } + return Message::unknown; + } + + /*! \brief Installs a new global message filter. + * \details Messages filtered by a global filter never reach the MessageHandlers. + * Local filters can be set on individual handlers. + * \param[in] filter The new filter to install + */ + void Logger::connectFilter(const ConstMessageFilter_ptr& filter) { + OES_ASSERT(filter, return;); + filters_.push_back(filter); + } + + /*! \brief Removes a global message filter + * \details Messages filtered by a global filter never reach the MessageHandlers. + * Local filters can be set on individual handlers + * \param[in] filter The filter to be removed (may not be null) + * \see connectFilter(ConstMessageFilter_ptr) + * \return true if the filter was removed + */ + bool Logger::disconnectFilter(const ConstMessageFilter_ptr& filter) { + OES_ASSERT(filter, return false;); + if (filters_.empty()) { + return false; + } + auto itr = std::remove(filters_.begin(), filters_.end(), filter); + if (itr == filters_.end()) { + return false; + } + filters_.erase(itr,filters_.end()); + return true; + } + + /*! \brief Removes all global filters + * \see connectFilter(ConstMessageFilter_ptr) + */ + void Logger::disconnectFilters() { + filters_.clear(); + } + + /*! + * \brief Pushes a new message context + * \details + * Message contexts are applied in reverse order from the last one to + * the first one. + * \param[in] context a message context + */ + void Logger::pushContext(const ConstMessageContext_ptr& context) { + OES_ASSERT(context, return;); + contexts_.push_back(context); + } + + /*! + * \brief Removes the last message context + */ + void Logger::popContext() { + contexts_.pop_back(); + } + + /*! + * \brief Clears all the message contexts + */ + void Logger::clearContexts() { + contexts_.clear(); + } + + /*! + * \brief Attaches a new handler to the stream. + */ + void Logger::connectHandler(const MessageHandler_ptr& handler) { + OES_ASSERT(handler, return;); + handlers_.push_back(handler); + } + + /*! + * \brief Removes the specified handler from the stream. + */ + void Logger::disconnectHandler(const MessageHandler_ptr& handler) { + OES_ASSERT(handler, return;); + handlers_.erase( + std::remove(handlers_.begin(), handlers_.end(), handler), + handlers_.end() + ); + } + + /*! + * \brief Resets the message stream to its initial state (Removes all + * handlers from the stream). + */ + void Logger::disconnectHandlers() { + handlers_.clear(); + } + + /*! + * \brief Sends the message to the logger. + * \details Message is dispatched to all registered handlers unless filtered + * by a global filter. + * If no handlers are registered, the message is sent on the standard + * output and to the debug window on Windows. + * All filters are provided with the opportunity to filter the messages. + */ + void Logger::send(const Message_ptr& message) { + // Check if the message is filtered globally + + bool discarded = false; + for (const ConstMessageFilter_ptr& filter : filters_) { + // filter cannot be null. See assert in connectFilter() + if (filter->filter(message)) { + discarded = true; + // do not return early, a filter may need to 'see' a message even if a previously set + // filter discards it. + } + } + if (discarded) { + return; + } + + // Apply the message contexts in reverse order + + for (const ConstMessageContext_ptr& context : reverse_of(contexts_)) { + // Contexts cannot be null (see assertion in pushContext()) + context->apply(message); + } + + // Dispatch messages to the handlers + + for (const MessageHandler_ptr& handler : handlers_) { + // Handlers cannot be null (see assertion in connectHandler()) + handler->receive(message); + } + + //Pass the messages to the device observer sink. The observer is set on the PG Logger initialization stage. + //The current logger is only the facade, the logging is done by the PG Logger. + //If the observer is not set the current logger is not functional. + if (messageSendObserver_.empty() || message->severity() > Message::last_log_severity) { + return; + } + oes::core::String text = pgLoggerFormatter_.format(message); + const oes::core::String::size_type text_length = text.length(); + if (text_length >=1 && '\n' == text.at(text_length - 1)) { + text.erase(text_length-1); + } + messageSendObserver_( + message->severity(), + "SharedComponents", + message->file().c_str(), + message->function().c_str(), + static_cast(message->lineNumber()), + text.c_str() + ); + } + + /*! + * \brief Creates a message filter based on a minimum severity + * + * The two highest levels, \em notice and \em status are special messages + * routed to custom UI components typically, and may not be appropriate + * to output in all contexts (e.g. unit tests). In such cases, using a + * maximum severity of \em fatal, to filter out those notice and status messages. + * + * \param minimumSeverity the minimum severity level a message must have (inclusive) + * \param maximumSeverity (Optional; defaults to max_severity, use last_log_severity to filter + * the messages specific to the UI) the maximum severity level a message may have (inclusive). + * \return the newly created filter; called is responsible for setting it on the + * Logger singleton or a specific MessageHandler. + */ + ConstMessageFilter_ptr Logger::createSeverityFilter( + Message::Severity minimumSeverity, + Message::Severity maximumSeverity + ) { + return boost::make_shared(minimumSeverity, maximumSeverity); + } + + /*! \brief Creates a handler sending messages to std::cout + * \details Uses by default a text formatter for its messages. The UI messages + * (severity > last_log_severity) and info/debug are discarded. + */ + MessageHandler_ptr Logger::createConsoleHandler() { + return boost::make_shared(); + } + + /*! \brief Creates a handler sending messages to the debugger console (if any) + * \details Currently a no-op handler on Linux or if debugger is not attached + */ + MessageHandler_ptr Logger::createDebugConsoleHandler() { + return boost::make_shared(); + } + + /*! + * \brief Gets the console handler, if any. + * + * Allows to configure the console handler's filters or formatter, + * without resorting to a global filter suppressing messages even before + * handlers see them. + * + * Especially useful for unit tests, so Notice and Status messages are + * not output to the console, yet test can still capture them via their + * own MessageHandler. + * + * \return the (first) console handler found; otherwise null, + * if no console handler was added using createConsoleHandler() + */ + MessageHandler_ptr Logger::consoleHandler() const { + for (const auto& handler : handlers_) { + if (dynamic_cast(handler.get())) { + return handler; + } + } + return MessageHandler_ptr(); // not found + } + + /*! + * \brief Gets the console handler, if any. + * + * Allows to configure the console handler's filters or formatter, + * without resorting to a global filter suppressing messages even before + * handlers see them. + * + * Especially useful for unit tests, so Notice and Status messages are + * not output to the console, yet test can still capture them via their + * own MessageHandler. + * + * \return the (first) console handler found; otherwise null, + * if no console handler was added using createConsoleHandler() + */ + MessageHandler_ptr Logger::debugConsoleHandler() const { + for (const auto& handler : handlers_) { + if (dynamic_cast(handler.get())) { + return handler; + } + } + return MessageHandler_ptr(); // not found + } + + } // namespace core +} // namespace oes + diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Logger.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Logger.h new file mode 100644 index 0000000000000000000000000000000000000000..9540e2295ccc676707fe2dd992c8123e4c287a4c --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Logger.h @@ -0,0 +1,445 @@ +// ============================================================================ +// Copyright 2008-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_LOGGER_H +#define OES_CORE_UTILS_LOGGER_H + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/*****************************************************************************/ +#if defined(_WIN32) && defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable:4275) // For 'boost::noncopyable' used as base for dll-interface +#endif + +namespace boost { + namespace program_options { + class options_description; + class variables_map; + } + class any; +} + +namespace oes { + namespace core { + + class OES_CORE_UTILS_EXPORT Message { + public: + enum Severity { + unknown = 0, + debug, + verbose, + info, + warning, + alert, + error, + critical, + fatal, + status, + notice, + cursor_notice, + first_log_severity = debug, + last_log_severity = fatal, + min_severity = debug, + max_severity = cursor_notice + }; + + static const String severityToString(Message::Severity severity); + static Message::Severity severityFromString(const String& severity); + + typedef boost::posix_time::ptime Time; + + virtual ~Message(); + + virtual Severity severity() const; + + virtual const String& prefix() const; + + Format format() const; + void setFormat(const Format& format); + + const Time& time() const; + String time_string() const; + + const String& function() const; + void setFunction(const String& function); + + const String& file() const; + void setFile(const String& file); + + size_t lineNumber() const; + void setLineNumber(size_t lineNumber); + + size_t depth() const; + void setDepth(size_t depth); + + protected: + Message(const Format& format); + + private: + Time time_; + String function_; + String file_; + size_t lineNumber_; + size_t depth_; + Format format_; + }; + + class OES_CORE_UTILS_EXPORT FatalMessage : public Message { + public: + FatalMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT CriticalMessage : public Message { + public: + CriticalMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT AlertMessage : public Message { + public: + AlertMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT ErrorMessage : public Message { + public: + ErrorMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT WarningMessage : public Message { + public: + WarningMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT NoticeMessage : public Message { + public: + NoticeMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT CursorNoticeMessage: public Message { + public: + CursorNoticeMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT InfoMessage : public Message { + public: + InfoMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT VerboseMessage : public Message { + public: + VerboseMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT DebugMessage : public Message { + public: + DebugMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT StatusMessage : public Message { + public: + StatusMessage(const Format& format); + virtual Severity severity() const; + virtual const String& prefix() const; + }; + + class OES_CORE_UTILS_EXPORT NotificationMessagesBufferEvent : public Message { + public: + struct turn_buffering_on_tag { }; + struct turn_buffering_off_tag { }; + + public: + NotificationMessagesBufferEvent(const turn_buffering_on_tag&); + NotificationMessagesBufferEvent(const turn_buffering_off_tag&); + + bool turn_buffering_on() const; + + private: + NotificationMessagesBufferEvent(const NotificationMessagesBufferEvent&); + NotificationMessagesBufferEvent& operator=(const NotificationMessagesBufferEvent&); + + private: + const bool turn_buffering_on_; + }; + + /***************************************************************************/ + + class OES_CORE_UTILS_EXPORT NotificationMessagesScopedBuffering { + public: + NotificationMessagesScopedBuffering(); + ~NotificationMessagesScopedBuffering(); + }; + + /***************************************************************************/ + + class OES_CORE_UTILS_EXPORT MessageFormatter : boost::noncopyable { + public: + virtual ~MessageFormatter(); + virtual String format(Message_ptr message) const = 0; + protected: + MessageFormatter() {} + }; + + class OES_CORE_UTILS_EXPORT XmlMessageFormatter : public MessageFormatter { + public: + XmlMessageFormatter() {} + virtual String format(Message_ptr message) const; + }; + + class OES_CORE_UTILS_EXPORT TextMessageFormatter : public MessageFormatter { + public: + TextMessageFormatter(const String& format = String()); + virtual String format(Message_ptr message) const; + private: + String format_; + }; + + /***************************************************************************/ + + class OES_CORE_UTILS_EXPORT MessageFilter : boost::noncopyable { + public: + virtual ~MessageFilter(); + virtual bool filter(Message_ptr message) const = 0; + protected: + MessageFilter() {} + }; + + typedef std::vector MessageFilterList; + + /***************************************************************************/ + + class OES_CORE_UTILS_EXPORT MessageContext : boost::noncopyable { + public: + virtual ~MessageContext(); + virtual Message_ptr apply(Message_ptr message) const = 0; + protected: + MessageContext() {} + }; + + typedef std::vector MessageContextList; + + /***************************************************************************/ + + class OES_CORE_UTILS_EXPORT MessageHandler : boost::noncopyable { + public: + virtual ~MessageHandler(); + void receive(const Message_ptr& message); + void connectFilter(const ConstMessageFilter_ptr& filter); + bool disconnectFilter(const ConstMessageFilter_ptr& filter); + void disconnectFilters(); + bool isFiltered(const Message_ptr& message); + + virtual bool setFormatter(ConstMessageFormatter_ptr formatter); + protected: + MessageHandler(bool useSeverityFilter = true); + virtual void handleMessage(const Message_ptr& message) = 0; + private: + MessageFilterList filters_; + }; + + typedef std::vector MessageHandlerList; + + /****************************************************************************/ + + class OES_CORE_UTILS_EXPORT Logger : public Singleton { + public: + Logger(); + virtual ~Logger(); + + bool configure(int argc, char *argv[]); + + template + void send( + const Format& format, + const String& file, + const int lineNumber, + const String& function = String() + ) { + Message_ptr message(new MessageType(format)); + message->setFile(file); + message->setLineNumber(lineNumber); + message->setFunction(function); + send(message); + } + + void send(const Message_ptr& message); + + void connectFilter(const ConstMessageFilter_ptr& filter); + bool disconnectFilter(const ConstMessageFilter_ptr& filter); + void disconnectFilters(); + + void connectHandler(const MessageHandler_ptr& handler); + void disconnectHandler(const MessageHandler_ptr& handler); + void disconnectHandlers(); + + void pushContext(const ConstMessageContext_ptr& context); + void popContext(); + void clearContexts(); + + // Built-in filter(s) + static ConstMessageFilter_ptr createSeverityFilter( + Message::Severity minimumSeverity, + Message::Severity maximumSeverity = Message::max_severity + ); + void setSeverityFilter( + Message::Severity minimumSeverity, + Message::Severity maximumSeverity = Message::max_severity + ); + Message::Severity minSeverityFilter() const; + Message::Severity maxSeverityFilter() const; + ConstMessageFilter_ptr severityFilter() const; + + // Built-in handler(s) + static MessageHandler_ptr createConsoleHandler(); + static MessageHandler_ptr createDebugConsoleHandler(); + + MessageHandler_ptr consoleHandler() const; + MessageHandler_ptr debugConsoleHandler() const; + + typedef boost::function MessageSendObserver; + + static void connectMessageSendObserver(MessageSendObserver); + + struct OES_CORE_UTILS_EXPORT ProgramOptions { + static Message::Severity validateSeverity(const std::vector&); + static const char* logSeverity_; + }; + + private: + friend class NotificationMessagesScopedBuffering; + ConstMessageFilter_ptr severity_filter_; + MessageFilterList filters_; + MessageContextList contexts_; + MessageHandlerList handlers_; + const TextMessageFormatter pgLoggerFormatter_; + + static MessageSendObserver messageSendObserver_; + }; + +#define OES_FATAL(format) OES_LOG(oes::core::FatalMessage, format) +#define OES_CRITICAL(format) OES_LOG(oes::core::CriticalMessage, format) +#define OES_ALERT(format) OES_LOG(oes::core::AlertMessage, format) +#define OES_ERROR(format) OES_LOG(oes::core::ErrorMessage, format) +#define OES_WARNING(format) OES_LOG(oes::core::WarningMessage, format) +#define OES_NOTICE(format) OES_LOG(oes::core::NoticeMessage, format) +#define OES_CURSOR_NOTICE(format) OES_LOG(oes::core::CursorNoticeMessage, format) +#define OES_INFO(format) OES_LOG(oes::core::InfoMessage, format) +#define OES_VERBOSE(format) OES_LOG(oes::core::VerboseMessage, format) +#define OES_DEBUG(format) OES_LOG(oes::core::DebugMessage, format) +#define OES_STATUS(format) OES_LOG(oes::core::StatusMessage, format) + +#if defined(_WIN32) +#define OES_LOG(type, format) \ + oes::core::Logger::instance().send(format, __FILE__, __LINE__) +#else +#define OES_LOG(type, format) \ + oes::core::Logger::instance().send(format, __FILE__, __LINE__, __FUNCTION__) +#endif + + /****************************************************************************/ + + inline const Message::Time& Message::time() const { + return time_; + } + + inline const String& Message::file() const { + return file_; + } + + inline void Message::setFile(const String& file) { + file_ = file; + } + + inline const String& Message::function() const { + return function_; + } + + inline void Message::setFunction(const String& function) { + function_ = function; + } + + inline size_t Message::lineNumber() const { + return lineNumber_; + } + + inline void Message::setLineNumber(size_t lineNumber) { + lineNumber_ = lineNumber; + } + + inline size_t Message::depth() const { + return depth_; + } + + inline void Message::setDepth(size_t depth) { + depth_ = depth; + } + + inline std::ostream& operator << (std::ostream& os, const Message& message) { + return os << message.format(); + } + + inline void Logger::connectMessageSendObserver (MessageSendObserver messageSendObserver) { + messageSendObserver_ = messageSendObserver; + } + + } // namespace core +} // namespace oes + +#if defined(_WIN32) && defined(_MSC_VER) +#pragma warning(pop) +#endif + +#endif + diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ObservableObserverImpl.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ObservableObserverImpl.h new file mode 100644 index 0000000000000000000000000000000000000000..3229958d28e4e5117fb589e0620f6d5b752daced --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/ObservableObserverImpl.h @@ -0,0 +1,547 @@ +// ============================================================================ +// Copyright 2011-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_OBSERVABLEOBSERVERIMPL_H +#define OES_CORE_UTILS_OBSERVABLEOBSERVERIMPL_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace oes { +namespace core { + + template class ObservableImpl; + template class ObserverImpl; + + /*! \brief A base class for typed observable objects providing auto-disconnect features + * \ingroup oes_core_utils + * \details + * \par What is it? + * - The ObservableImpl is a template class providing (a possible) interface and implementation of + * the Observer/Observable pattern. + * - The ObservableImpl is meant to be used as a (public) base class of the type being observed + * (i.e. class MyObservable : public ObservableImpl ). + * - The ObservableImpl works in conjunction with its observer counter part (ObserverImpl), + * which follows the same logic + * (i.e. class MyObserver : public ObserverImpl ). + * \par What does it provide? + * - Provides the interface to connect/disconnect observers, along with the notification + * mechanism. Several callbacks (onConnect, onDisconnect, onNotify) are provided. + * - Provides auto-disconnection of observers upon destruction of the observable value, + * a special callback (onDestroy) being provided for that effect. + * - Supports nested connection/disconnection/notification, meaning that (i) new observers can + * be connected within disconnection/notification loops (ii) observers can be + * disconnected within connection/notification loops and so on... + * \note This class's intent is not to provide an universal replacement for all Observable + * interfaces and implementation, It is meant to avoid a lot of boiler-plate, yet very + * error-prone, code. + * \note The observers connected to an observable object are stored as a sequence (not as a set) + * to ensure notification order is reproducible. + */ + template + class ObservableImpl : boost::noncopyable { + + public: + typedef T ObservableType; + typedef ObserverImpl ObserverType; + + /*! \brief Destructor + * \details All connected observers are automatically disconnected and notified of + * deletion of the observable + * \see ObserverImpl::onDestroy + * \pre You must not delete the observable within a notification loop + */ + virtual ~ObservableImpl() { + OES_ASSERT(notifyDepth_==0, return;); + blocked_ = true; // Make sure no attempt is made to connect/disconnect during destruction + if( observers_ ) { + for(typename ObserverSequence::iterator itr = observers_->begin(); itr != observers_->end(); + ++itr) { + OES_ASSERT(*itr!=nullptr, return;); + (*itr)->handleOnDestroy(getSelfAsObservable()); + } + } + } + + /*! \brief Connects a new observer and returns true if succeeded + * \details + * - If the input observer is already connected, this is a no-op and true is returned + * (not onConnect callback is triggered). + * - If the input observer is not already connected, it is inserted in the list of + * observers, and onConnect callback is triggered only if triggerOnConnect is true. + * Null input observers are discarded, false is returned. + * - If you attempt to connect a new observer while the observable is being destroyed + * (you really should not), false is returned. + * - If you attempt to connect a new observer while the observable is processing a + * disconnectAll request, false is returned. + * - You may safely connect new observers while the observable is processing a + * notify request (newly connected observers will receive BOTH on onConnect and a onNotify + * callback). + * \param[in] observer The observer to be connected + * \param[in] triggerOnConnect If true, a onConnect notification is triggered upon successful connection + * \returns true if the connection succeeded, false otherwise + * \note About const-correctness: this function is considered 'const' as attaching a new + * observer does not change the externally visible state of the observable object. + * \see ObserverImpl::onConnect + */ + bool connect(ObserverType* observer, bool triggerOnConnect = true) const { + if( not observer or blocked_ ) { + return false; // NULL observers not allowed, Modifying list of observers is not allowed (for now) + } + + if( not observers_ ) { + // List of observers not instantiated yet, there is no need + // to double-check if input observer is already connected. + observers_.reset(new ObserverSequence); + observers_->push_back(observer); + } else { + // List of observers already instantiated, we need + // to check that input observer is not already connected. + typename ObserverSequence::const_iterator itr = std::find( + observers_->begin(), observers_->end(), observer + ); + if( itr != observers_->end() ) { + return true; // Already there, no need to trigger onConnect + } + observers_->push_back(observer); + } + if( triggerOnConnect ) { + observer->handleOnConnect(getSelfAsObservable()); + } + return true; + } + + /*! \brief Disconnects an existing observer and returns true if succeeded + * \details + * - If the input observer is not connected, this is a no-op and false is + * returned. + * - If the input observer is connected, it is removed from the list of + * observers, and onDisconnect callback is triggered only if triggerOnDisconnect is true. + * - If you attempt to disconnect a observer while the observable is being destroyed + * (you really should not), false is returned. + * - If you attempt to disconnect a observer while the observable is processing a + * disconnectAll request, false is returned. + * - You may safely disconnect observers while the observable is processing a + * notify request. + * \param[in] observer The observer to be disconnected + * \param[in] triggerOnDisconnect If true, a onDisconnect notification is triggered upon successful disconnection + * \returns true if the disconnection succeeded, false otherwise + * \note About const-correctness: this function is considered 'const' as attaching a new + * observer does not change the externally visible state of the observable object. + * \see ObserverImpl::onDisconnect + */ + bool disconnect(ObserverType* observer, bool triggerOnDisconnect = true) const { + if( not observers_ ) { + return false; // Nothing to do + } + if( not observer or blocked_ ) { + return false; // NULL observers not allowed, Modifying list of observers is not allowed (for now) + } + + typename ObserverSequence::iterator itr = std::find( + observers_->begin(), observers_->end(), observer + ); + if( itr == observers_->end() ) { + return false; // Not connected + } + if( notifyDepth_ > 0 ) { + // When observers are removed during a notification loop, + // we should not shrink the list. We just zero-out the entry + *itr = nullptr; + } else { + // When observers are removed outside of a notification loop, it is + // safe to manipulate the sequence directly. + observers_->erase(itr); + } + + if( triggerOnDisconnect ) { + observer->handleOnDisconnect(getSelfAsObservable()); + } + return true; + } + + /*! \brief Disconnects ALL an existing observers + * \details + * - Each individual observers are removed from the list of observers and receive a + * onDisconnect notification. + * - A notable side-effect of this function is that no new observers can be connected while + * within this function (connect will return false). This guarantees that the observable is no + * longer observed upon function exit. + * \see ObserverImpl::onDisconnect + */ + void disconnectAll() { + if( not observers_ ) { + return; // Nothing to do + } + if( blocked_ ) { + return; // Modifying list of observers is not allowed (for now) + } + + blocked_ = true; // Forbid new connections + try { + for(typename ObserverSequence::iterator itr = observers_->begin(); itr != observers_->end(); + ++itr) { + if( *itr != nullptr ) { + (*itr)->handleOnDisconnect(getSelfAsObservable()); + (*itr) = nullptr; + } + } + if( notifyDepth_ == 0 ) { + // We make sure to only delete the list when outside of a notification loop, + // even if disconnectAll() would/should "normally" not be called within a notification + // loop (disconnectAll() is not a const function while notify() is). + observers_.reset(); + } + } catch(...) { + // This is where a finally clause would be really handy + if( notifyDepth_ == 0 ) { + // We make sure to remove any NULL entries but ONLY IF outside of a + // notification loop (if within a notification loop, this is taken care of + // by notify() directly). + cleanObserverSequence(); + } + blocked_ = false; + throw; // Re-throw, after making sure we do not lock the observable in permanent blocked state + } + blocked_ = false; + } + + /*! \brief Notify ALL an existing observers that the observable state has changed + * \details + * - You may safely connect new observers while the observable is processing a + * notify request. + * - You may safely disconnect existing observers while the observable is processing a + * notify request. + * \note About const-correctness: this function is considered 'const' as notifying + * observers does not (not supposed) change the externally visible state of the observable + * object. + * \see ObserverImpl::onNotify + */ + virtual void notify() const { + if( not observers_ ) { + return; // Nothing to do + } + + // It is important here to iterate using indices, rather + // than iterators. + // - If new observers are connected during this notification loop, + // the iterators will be invalidated (see push_back in connect) and the + // actual size might grow. + // - If observers are disconnected during this notification loop, special + // care is taken in disconnect so not to shrink the list. Instead the entry + // is zero-ed out (hence the NULL check below). + notifyDepth_++; + try { + for(std::size_t i = 0; i < observers_->size(); ++i) { + ObserverType* observer = observers_->at(i); + if( observer ) { + observer->handleOnNotify(getSelfAsObservable()); + } + } + } catch(...) { + notifyDepth_--; + if( notifyDepth_==0 ) { + cleanObserverSequence(); + } + throw; // Re-throw, after making sure we do not lock the observable in illegal state + } + notifyDepth_--; + if( notifyDepth_==0 ) { + // When exiting the nested notification loops, make sure we removed + // any zero-ed entry. + cleanObserverSequence(); + } + } + + protected: + /*! \brief Constructor */ + ObservableImpl() : observers_(), blocked_(false), notifyDepth_(0) { + } + /*! \brief Returns this casted as the target (leaf class) observable object */ + const ObservableType* getSelfAsObservable() const { + // If you get a compilation error here, it most likely means you did not use properly + // the ObservableImpl/ObserverImpl system. The template argument T of ObservableImpl + // should be convertible to a ObservableImpl. Something like: + // class MyObservable : public ObservableImpl {}; + // class MyObserver : public ObserverImpl {}; + return static_cast(this); // This will not compile either + } + + private: + /*! \brief Remove the entries that were zero-ed out during the notification loop(s) */ + void cleanObserverSequence() const { + OES_ASSERT(notifyDepth_==0, return;); + observers_->erase( + std::remove(observers_->begin(), observers_->end(), (ObserverType*)nullptr), + observers_->end() + ); + } + typedef std::vector ObserverSequence; + mutable boost::scoped_ptr observers_; + bool blocked_; + mutable std::size_t notifyDepth_; + }; + + /*! \brief A base class for typed observers objects providing auto-disconnect features + * \ingroup oes_core_utils + * \details + * \par What is it? + * - The ObserverImpl is a template class providing (a possible) interface and implementation of + * the Observer/Observable pattern. + * - The ObserverImpl is meant to be used as a (public) base class of the type observing + * (i.e. class MyObserver : public ObserverImpl ). + * - The ObserverImpl works in conjunction with its observable counter part (ObservableImpl), + * which follows the same logic + * (i.e. class MyObservable : public ObservableImpl ). + * \par What does it provide? + * - Provides a special (derivation) interface for concrete classes to be notified when + * new objects are connected, existing are disconnected/destroyed and updated. + * - Keeps track of connected Observables and auto-disconnect from these itself when + * it is being destroyed. + * \par Notifications + * The 4 following virtual functions are provided for derived classes + * - onNotify, called whenever one of the observable objects has changed and was notified + * - onConnect, called whenever a new observable object was connected to the observer + * - onDisconnect, called whenever one of the observable objects was disconnected + * - onDestroy, called whenever one of the observable objects is being destroyed + * \note This class's intent is not to provide an universal replacement for all Observer + * interfaces and implementation, It is meant to avoid a lot of boiler-plate, yet very + * error-prone, code. + */ + template + class ObserverImpl : boost::noncopyable { + + public: + typedef T ObservableType; + typedef ObserverImpl ObserverType; + + /*! \brief Destructor, all connected observables are automatically disconnected */ + virtual ~ObserverImpl() { + disconnectFromAll(); + } + + protected: + /*! \brief Constructor */ + ObserverImpl() : observables_(), blockedObservables_() { + } + + + /*! \brief Called whenever one of the observed objects has changed and was notified + * \details This function is however NOT called if the input observable object + * has been "blocked". Default implementation does nothing. + * \see blockNotifyFrom + */ + virtual void onNotify(const ObservableType* observable) { + OES_DEBUG_ASSERT(not isNotifyBlockedFrom(observable)); + // Does nothing by default + OES_UNUSED(observable); + } + /*! \brief Called whenever a new observable objects is connected + * \details Default implementation does nothing. This function is called after the + * new observable connection, so disconnectFrom(observable) is true when entering + * this function. + */ + virtual void onConnect(const ObservableType* /*observable*/) { + // Does nothing by default + } + /*! \brief Called whenever one of the observed objects is disconnected + * \details Default implementation does nothing. This function is called after the + * new observable disconnection, so disconnectFrom(observable) is false when entering + * this function. + */ + virtual void onDisconnect(const ObservableType* /*observable*/) { + // Does nothing by default + } + /*! \brief Called whenever one of the observed objects is being destroyed + * \details Default implementation does nothing. This function is called after the + * new observable disconnection, so disconnectFrom(observable) is false when entering + * this function. + * \warning Remember that the input observable object is being destroyed (this is typically + * triggered within ObservableImpl::~ObservableImpl), and that you should attempt to manipulate + * (access member functions). + */ + virtual void onDestroy(const ObservableType* /*observable*/) { + // Does nothing by default + } + + /*! \brief Returns true if the input observable is connected to self + * \param[in] observable The observable object + */ + bool isConnectedTo(const ObservableType* observable) const { + return observables_ && observables_->find(observable) != observables_->end(); + } + + /*! \brief Disconnects self from the input observable and returns true if succeeded + * \details + * - This is the preferred and safer way to disconnect itself from any observable + * objects. + * - Since the disconnection is initiated by the observer itself, no onDisconnect + * notification will be sent from the observable to the observer or to derived classes. + * - If the input observable was not connected to this observer, this is a no-op + * and false is returned. + * \param[in] observable The observable object to be disconnected from + * \returns true if the input observable was successfully disconnected, false otherwise. + */ + bool disconnectFrom(const ObservableType* observable) { + if( not observables_ or observables_->erase(observable)==0 ) { + OES_DEBUG_ASSERT(not isNotifyBlockedFrom(observable)); // If the observable is not connected, it can't have a blocked status + return false; // Not connected + } + if( blockedObservables_ ) { + blockedObservables_->erase(observable); + } + return observable->disconnect(this, false /* no need to notify self */); + } + + /*! \brief Disconnects self from ALL the observables + * \details + * - This is the preferred and safer way to disconnect itself from all observable + * objects. + * - Since the disconnection is initiated by the observer itself, no onDisconnect + * notification will be sent from the observable to the observer or to derived classes. + */ + void disconnectFromAll() { + if( observables_ ) { + for(typename ObservableSet::iterator itr = observables_->begin(); itr != observables_->end(); + ++itr) { + (*itr)->disconnect(this, false /* no need to notify self */); + } + observables_.reset(); + } + blockedObservables_.reset(); + } + + /*! \brief Blocks or un-blocks notification from the input observable object + * \details + * - If input \p newStatus is \c true (the default), the observer will no longer receive + * notifications from the input observable object, \c onNotify will no longer + * be called. + * - If input \p newStatus is \c false, the observer will receive again + * notifications from the input observable object, \c onNotify will be called + * again. + * - This only applies to \c onNotify, not to any of the other type of + * notification such as \c onConnect, \c onDisconnect or \c onDestroy + * \param[in] observable The observable object + * \param[in] newStatus The new blocking status + * \return The OLD/PREVIOUS blocking status for this input observable object + * \pre The observer must be connected to the input observable + */ + bool blockNotifyFrom(const ObservableType* observable, bool newStatus = true) { + OES_DEBUG_ASSERT(isConnectedTo(observable)); + if( newStatus ) { + // Turn OFF notifications, blocked + if( not blockedObservables_ ) { + blockedObservables_.reset(new ObservableSet); + } + return not blockedObservables_->insert(observable).second; + } + // Turn ON notifications, unblocked + return blockedObservables_ && blockedObservables_->erase(observable) > 0; + } + + /*! \brief Un-blocks notification from the input observable object + * \details This function essentially behaves as \c blockNotifyFrom with an input + * \p newStatus of false. + * \param[in] observable The observable object + * \return The OLD/PREVIOUS blocking status for this input observable object + * \pre The observer must be connected to the input observable + */ + bool unblockNotifyFrom(const ObservableType* observable) { + return blockNotifyFrom(observable, false); + } + + /*! \brief Returns true if the notification from the input observable object are currently blocked + * \param[in] observable The observable object + */ + bool isNotifyBlockedFrom(const ObservableType* observable) const { + return blockedObservables_ && blockedObservables_->find(observable) != blockedObservables_->end(); + } + + private: + friend class ObservableImpl; + + /*! \brief Handles notification received from ObservableImpl + * \param[in] observable One the observable objects being observed by self + * \pre isConnectedTo(observable) must be true + */ + void handleOnNotify(const ObservableType* observable) { + OES_DEBUG_ASSERT(isConnectedTo(observable)); + if( not isNotifyBlockedFrom(observable) ) { + onNotify(observable); + } + } + + /*! \brief Handles connection notification received from ObservableImpl + * \param[in] observable A new observable objects to be observed by self + * \pre isConnectedTo(observable) must be false + */ + void handleOnConnect(const ObservableType* observable) { + OES_DEBUG_ASSERT(not isConnectedTo(observable) and not isNotifyBlockedFrom(observable)); + if( not observables_ ) { + observables_.reset(new ObservableSet); + } + observables_->insert(observable); + onConnect(observable); + } + + /*! \brief Handles disconnection notification received from ObservableImpl + * \param[in] observable One the observable objects being observed by self + * \pre isConnectedTo(observable) must be true + */ + void handleOnDisconnect(const ObservableType* observable) { + OES_DEBUG_ASSERT(isConnectedTo(observable)); + if( observables_ ) { + observables_->erase(observable); + if( blockedObservables_ ) { + blockedObservables_->erase(observable); + } + onDisconnect(observable); + } + } + + /*! \brief Handles destruction notification received from ObservableImpl + * \param[in] observable One the observable objects being observed by self + * \pre isConnectedTo(observable) must be true + */ + void handleOnDestroy(const ObservableType* observable) { + OES_DEBUG_ASSERT(isConnectedTo(observable)); + if( observables_ ) { + observables_->erase(observable); + if( blockedObservables_ ) { + blockedObservables_->erase(observable); + } + onDestroy(observable); + } + } + + typedef std::set ObservableSet; + mutable boost::scoped_ptr observables_; + mutable boost::scoped_ptr blockedObservables_; + }; + +} // namespace core +} // namespace oes + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Preprocessor.h b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Preprocessor.h new file mode 100644 index 0000000000000000000000000000000000000000..fa5decb770edd3e3cd9a3d370d9ef38977074b93 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/Preprocessor.h @@ -0,0 +1,80 @@ +// ============================================================================ +// Copyright 2011-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_PREPROCESSOR_H +#define OES_CORE_UTILS_PREPROCESSOR_H + +#include +#include +#include + +/*! \brief Macro to embed pragma compiler directives within other macros + * \details + * - Pragmas cannot be typically embedded within macros as it contains + * special characters, workaround is compiler-dependent. + * - This is meant to provide such workaround transparently, using + * available facilities. + * \ingroup oes_core_utils + */ +#if defined(_WIN32) + // Visual VC + #define OES_PP_PRAGMA(x) __pragma(x) +#else + // Intel ICC and GNU CC, using ANSI C99 operator + #define OES_PP_PRAGMA(x) _Pragma(#x) +#endif + +/*! \brief Empty macro except when using Intel compiler + * \ingroup oes_core_utils + */ +#if defined(__INTEL_COMPILER) + #define OES_PP_IF_INTEL(x) x +#else + #define OES_PP_IF_INTEL(x) +#endif + +/*! \brief Empty macro except when using Microsoft VC compiler + * \ingroup oes_core_utils + */ +#if defined(_WIN32) // Not using _MSC_VER, as within SharedComponents, we equate WIN32 to VC + #define OES_PP_IF_MSVC(x) x +#else + #define OES_PP_IF_MSVC(x) +#endif + +/*! \brief Empty macro except when using GNU GCC compiler + * \ingroup oes_core_utils + */ +#if defined(__GNUC__) and not defined(__INTEL_COMPILER) + #define OES_PP_IF_GCC(x) x +#else + #define OES_PP_IF_GCC(x) +#endif + +/*! \brief Provides a unique identifier from an input prefix + * \details + * - Essentially meant to generate identifiers (notably for local/static) variable + * names within other macros. + * \par Known limitations: + * - The generated identifier is essentially based on __LINE__ ANSI predefined + * macro until __COUNTER__ becomes more widely available. + * - As a consequence, given a same prefix, the macro may expand to the same + * identifier if you use it several times within the same line. Use a different prefix. + * \ingroup oes_core_utils + */ +#define OES_PP_UNIQUE_ID(Prefix) BOOST_PP_CAT(Prefix,__LINE__) + +#endif diff --git a/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/RangeUtils.hpp b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/RangeUtils.hpp new file mode 100644 index 0000000000000000000000000000000000000000..34f3a95304dcdff2fba7c2ab18fac894c549f3a7 --- /dev/null +++ b/Reservoir DMS Suite/open-etp-server/src/lib/oes/core/utils/RangeUtils.hpp @@ -0,0 +1,94 @@ +// ============================================================================ +// Copyright 2015-2021 Emerson Paradigm Holding LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef OES_CORE_UTILS_RANGEUTILS_HPP +#define OES_CORE_UTILS_RANGEUTILS_HPP + +#include +#include +#include +#include