# -*- mode: qore; indent-tabs-mode: nil -*-
#! Qore AbstractDataProvider class definition

/** AbstractDataProvider.qc Copyright 2019 - 2023 Qore Technologies, s.r.o.

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

%requires(reexport) Logger

#! contains all public definitions in the DataProvider module
public namespace DataProvider {
/** @defgroup db_provider_upsert_results DB Provider Upsert Result Codes
    These are the results codes returned with single-record upsert actions
*/
#/@{
#! Indicates that the record was inserted
public const UpsertResultInserted = "inserted";

#! Indicates that the record was updated
public const UpsertResultUpdated = "updated";

#! Indicates that the record was verified as already in the target state
public const UpsertResultVerified = "verified";

#! Indicates that the record was left unchanged
public const UpsertResultUnchanged = "unchanged";

#! Indicates that the record was deleted
public const UpsertResultDeleted = "deleted";
#/@}

#! Allowed value hash
public hashdecl AllowedValueInfo {
    #! The value
    auto value;

    #! Description for the value
    string desc;
}

#! Data provider option info
public hashdecl DataProviderOptionInfo {
    #! The option value type or types
    softlist<AbstractDataProviderType> type;

    #! Required flag
    bool required = False;

    #! The option description
    string desc;

    #! Any default value
    auto default_value;

    #! Is the option value sensitive?
    bool sensitive = False;

    #! List of allowed values (enum)
    *softlist<hash<AllowedValueInfo>> allowed_values;
}

#! Data provider summary info
public hashdecl DataProviderSummaryInfo {
    #! The name of the data provider
    string name;

    #! The description of the data provider; supports markdown
    string desc;

    #! The name of the provider type
    string type;

    #! Does the data provider support reading
    bool supports_read = False;

    #! Does the data provider support record creation?
    bool supports_create = False;

    #! Does the data provider support record updates?
    bool supports_update = False;

    #! Does the data provider support record upserts (create or update)?
    bool supports_upsert = False;

    #! Does the data provider support record deletion?
    bool supports_delete = False;

    #! Does the data provider support native record searching?
    bool supports_native_search = False;

    #! Does the data provider support native / optimized bulk reads?
    /** if this is @ref False and \a supports_read is @ref True, the data provider will still support bulk read
        APIs but without any performance improvements for bulk data (simulated bulk reads)
    */
    bool supports_bulk_read = False;

    #! Does the data provider support native / optimized bulk creation?
    /** if this is @ref False and \a supports_creation is @ref True, the data provider will still support bulk write
        APIs but without any performance improvements for bulk data (simulated bulk creates)
    */
    bool supports_bulk_create = False;

    #! Does the data provider support native / optimized bulk upserts?
    /** if this is @ref False and \a supports_upsert is @ref True, the data provider will still support bulk upsert
        APIs but without any performance improvements for bulk data (simulated bulk upserts)
    */
    bool supports_bulk_upsert = False;

    #! Does the data provider support the request API?
    bool supports_request = False;

    #! Does the data provider support children?
    bool supports_children = False;

    #! Does the data provider require transaction management?
    bool transaction_management = False;

    #! Does the data provider provide a record?
    bool has_record = False;

    #! Does this data provide support advanced search experssions?
    bool supports_search_expressions = False;

    #! Does this data provider support the event API / observer pattern?
    /**
        @since DataProvider 2.5
    */
    bool supports_observable = False;

    #! Message support
    /** This describes the data provider's support for output messages, if any, and the relationship to observed
        events, if any.

        @see dataprovider_message_support

        @since DataProvider 2.5
    */
    string supports_messages = MSG_None;

    #! Can any child data providers offer API services (request - response data providers)?
    /** @since DataProvider 2.7
    */
    bool children_can_support_apis = False;

    #! Can any child data providers offer record-based providers?
    /** @since DataProvider 2.7
    */
    bool children_can_support_records = False;

    #! Can any child data providers support the observer pattern / event API?
    /** @since DataProvider 2.7
    */
    bool children_can_support_observers = False;

    #! Can any child data providers support messages?
    /** @since DataProvider 2.7
    */
    bool children_can_support_messages = False;

    #! Can any child data providers support transaction management?
    /** @since DataProvider 2.8
    */
    bool children_can_support_transactions = False;
}

#! Data provider message and event info
public hashdecl DataProviderMessageInfo {
    #! The description of the message or event
    /** Markdown is supported
    */
    string desc;

    #! The data type of the message or event
    AbstractDataProviderType type;
}

/** @defgroup dataprovider_message_support Output Message Support

    These constants describe the data provider's support for output messages, if any, and the relationship to observed
    events, if any.

    @since DataProvider 2.5
*/
#/@{
#! No output type
/** The data provider does not support output messages
*/
public const MSG_None = "NONE";

#! Synchronous output type, an output message is expected after each observed event
/** The data provider supports output messages, and an output message is expected after each input message.

    This can be used to support the server-side request-response API integration pattern
*/
public const MSG_Sync = "SYNC";

#! Asynchronous output type, output can happen any time
/** Input (i.e. observed events) may not be related or supported
*/
public const MSG_Async = "ASYNC";
#/@}

/** @defgroup signature_type_codes Signature Type Codes

    @since DataProvider 2.3
*/
#/@{
#! Indicates any valid expression that returns the given type (default)
public const ST_Any = 1;

#! Indicates any literal value of the given type; i.e. not an operator or other expression
public const ST_Value = 2;

#! The signature type indicates a string field name when used as an argument, and the type is the field type
/** in this case the value must always be a string (referring to the field name), and the type is always "any",
    referring to the fact that the final type reflects the field's type, which can be of any supported type
*/
public const ST_Field = 3;
#/@}

#! Maps signature type codes to descriptions
public const SignatureTypeCodeMap = {
    ST_Any: "any",
    ST_Value: "value",
    ST_Field: "field reference",
};

#! Maps signature description values to integer codes
public const SignatureTypeDescMap = map {$1.value: $1.key.toInt()}, SignatureTypeCodeMap.pairIterator();

/** @defgroup expression_role_codes Expression Role Codes

    @since DataProvider 2.3
*/
#/@{
#! Indicates an expression that can be used in a search operation
public const ER_Search = (1 << 0);

#! Indicates an expression that can be used in a field
public const ER_Field = (1 << 1);

#! Indicates an expression that can be used in all contexts
public const ER_All = ER_Search | ER_Field;
#/@}

#! Role map from codes to descriptions
public const RoleCodeMap = {
    ER_Search: "search",
};

#! Role map from descriptions to codes
public const RoleDescMap = map {$1.value: $1.key.toInt()}, RoleCodeMap.pairIterator();

/** @defgroup logic_capability_codes Logic Capability Codes

    @since DataProvider 2.3
*/
#/@{
#! Indicates that boolean expressions can be combined with AND logic
public const LC_And = (1 << 0);

#! Indicates that boolean expressions can be combined with OR logic
public const LC_Or = (1 << 1);

#! Indicates that all logic capabilities are supported
public const LC_All = (LC_And | LC_Or);
#/@}

#! Logic map from codes to descriptions
public const LogicCodeMap = {
    LC_And: "AND",
    LC_Or: "OR",
};

#! Logic map from descriptions to integer codes
public const LogicDescMap = map {$1.value: $1.key.toInt()}, LogicCodeMap.pairIterator();

#! Data provider signature argument info
public hashdecl DataProviderSignatureTypeInfo {
    #! Argument types
    AbstractDataProviderType type;

    #! The signature type code
    int type_code = ST_Any;
}

#! Data provider signature field type
public const DataProviderSignatureFieldType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."any",
    "type_code": ST_Field,
};

#! Data provider signature any type
public const DataProviderSignatureAnyType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."any",
    "type_code": ST_Any,
};

#! Data provider signature string value type
public const DataProviderSignatureStringValueType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."string",
    "type_code": ST_Value,
};

#! Data provider signature optional string value type
public const DataProviderSignatureOptionalStringValueType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."*string",
    "type_code": ST_Value,
};

#! Data provider signature int value type
public const DataProviderSignatureIntValueType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."int",
    "type_code": ST_Value,
};

#! Data provider signature *int value type
public const DataProviderSignatureOptionalIntValueType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."*int",
    "type_code": ST_Value,
};

#! Data provider signature list value type
public const DataProviderSignatureListValueType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."list",
    "type_code": ST_Value,
};

#! Data provider signature hash value type
public const DataProviderSignatureHashValueType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."hash",
    "type_code": ST_Value,
};

#! Data provider signature bool type
public const DataProviderSignatureBoolType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."bool",
    "type_code": ST_Any,
};

#! Data provider signature string type
public const DataProviderSignatureStringType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."string",
    "type_code": ST_Any,
};

#! Data provider signature date type
public const DataProviderSignatureDateType = <DataProviderSignatureTypeInfo>{
    "type": AbstractDataProviderTypeMap."date",
    "type_code": ST_Any,
};

/** @defgroup dataprovider_expression_type_codes DataProvider Expression Type Codes

    @since DataProvider 2.3
*/
#/@{
#! Indicates an operator expression
public const DET_Operator = 1;

#! Indicates a function expression
public const DET_Function = 2;
#/@}

#! Maps DataProvider expression type codes to descriptions
public const ExpressionTypeCodeMap = {
    DET_Operator: "operator",
    DET_Function: "function",
};

#! Maps DataProvider expression type descriptions to integer codes
public const ExpressionTypeDescMap = map {$1.value: $1.key.toInt()}, ExpressionTypeCodeMap.pairIterator();

#! Data provider signature info
public hashdecl DataProviderExpressionInfo {
    #! The type of expression; see @ref dataprovider_expression_type_codes
    int type;

    #! The label used for the expression
    /** Must correspond to the key used in any expression hash
    */
    string label;

    #! The display name
    string name;

    #! The description of the operation
    string desc;

    #! The symbol to use when rendering expressions
    string symbol;

    #! The expression role code / bitfield
    /** @see @ref expression_role_codes
    */
    int role = ER_All;

    #! Argument types
    softlist<hash<DataProviderSignatureTypeInfo>> args;

    #! The return type
    AbstractDataProviderType return_type;

    #! The last argument can be repeated indefinitely
    bool varargs = False;
}

#! Data provider field reference
public hashdecl DataProviderFieldReference {
    #! The name of the field being referenced
    string field;
}

#! Data provider expression
public hashdecl DataProviderExpression {
    #! Expression name
    string exp;

    #! Expression arguments
    /** Either values, field references (hash<DataProviderFieldReference>), or further expressions
        (hash<DataProviderExpression> values) to be evaluated
    */
    softlist<auto> args;
}

#! Data provider info
public hashdecl DataProviderInfo {
    #! Data provider name
    /** @note this is a dynamic attribute; it differs for different data providers of the same type
    */
    string name;

    #! The name of the provider type
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    string type;

    #! Does the data provider support reading
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_read = False;

    #! Does the data provider support record creation?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_create = False;

    #! Does the data provider support record updates?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_update = False;

    #! Does the data provider support record upserts (create or update)?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_upsert = False;

    #! Does the data provider support record deletion?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_delete = False;

    #! Does the data provider support native record searching?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_native_search = False;

    #! Does the data provider support native / optimized bulk reads?
    /** if this is @ref False and \a supports_read is @ref True, the data provider will still support bulk read
        APIs but without any performance improvements for bulk data (simulated bulk reads)

        @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_bulk_read = False;

    #! Does the data provider support native / optimized bulk creation?
    /** if this is @ref False and \a supports_creation is @ref True, the data provider will still support bulk write
        APIs but without any performance improvements for bulk data (simulated bulk creates)

        @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_bulk_create = False;

    #! Does the data provider support native / optimized bulk upserts?
    /** if this is @ref False and \a supports_upsert is @ref True, the data provider will still support bulk upsert
        APIs but without any performance improvements for bulk data (simulated bulk upserts)

        @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_bulk_upsert = False;

    #! Does the data provider support the request API?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_request = False;

    #! Does the data provider support children?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool supports_children = False;

    #! Does the data provider require transaction management?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool transaction_management = False;

    #! Does the data provider provide a record?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool has_record = False;

    #! Do we require search options to retrieve the record type?
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    bool record_requires_search_options = False;

    #! Does the data provider support creating new child providers?
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    bool supports_child_create = False;

    #! Does the data provider support deleting new child providers?
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    bool supports_child_delete = False;

    #! Does the data provider support adding new fields?
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    bool supports_add_field = False;

    #! Does the data provider support updating existing fields?
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    bool supports_update_field = False;

    #! Does the data provider support deleting fields?
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    bool supports_delete_field = False;

    #! Does the data provider support a schema?
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.2
    */
    bool supports_schema = False;

    #! Does this data provide support advanced search experssions?
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.2
    */
    bool supports_search_expressions = False;

    #! Does this data provider support the event API / observer pattern?
    /** If this is @ref True, then the data provider will inherit @ref Observable and can be observed

        @since DataProvider 2.5
    */
    bool supports_observable = False;

    #! Output message support
    /** @see dataprovider_message_support

        This attribute defines if the message API is supported by the data provider as well as the relationship to
        observable events, if any

        @since DataProvider 2.5
    */
    string supports_messages = MSG_None;

    #! Can any child data providers offer API services (request - response data providers)?
    /** @since DataProvider 2.6
    */
    bool children_can_support_apis = False;

    #! Can any child data providers offer record-based providers?
    /** @since DataProvider 2.6
    */
    bool children_can_support_records = False;

    #! Can any child data providers support the observer pattern / event API?
    /** @since DataProvider 2.6
    */
    bool children_can_support_observers = False;

    #! Can any child data providers support messages?
    /** @since DataProvider 2.7
    */
    bool children_can_support_messages = False;

    #! Can any child data providers support transaction management?
    /** @since DataProvider 2.8
    */
    bool children_can_support_transactions = False;

    #! Returns the schema type supported by this data provider
    /**
        @since DataProvider 2.2
    */
    *string schema_type;

    #! Constructor options supported by the data provider for the constructor variant taking a hash
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    *hash<string, hash<DataProviderOptionInfo>> constructor_options;

    #! Create options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    *hash<string, hash<DataProviderOptionInfo>> create_options;

    #! Upsert options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    *hash<string, hash<DataProviderOptionInfo>> upsert_options;

    #! Search options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    *hash<string, hash<DataProviderOptionInfo>> search_options;

    #! Request options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type
    */
    *hash<string, hash<DataProviderOptionInfo>> request_options;

    #! Child creation options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> child_create_options;

    #! Child deletion options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> child_delete_options;

    #! Add field options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> add_field_options;

    #! Update field options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> update_field_options;

    #! Delete field options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> delete_field_options;

    #! Send message options supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.5
    */
    *hash<string, hash<DataProviderOptionInfo>> send_message_options;

    #! Expressions (operators and functions) supported by the data provider
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.3
    */
    *hash<string, hash<DataProviderExpressionInfo>> expressions;

    #! hash of observable events and event types
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.5
    */
    *hash<string, hash<DataProviderMessageInfo>> events;

    #! List of output message IDs and message types
    /** @note this is a static attribute; it is the same for all data providers of the same type

        @since DataProvider 2.5
    */
    *hash<string, hash<DataProviderMessageInfo>> messages;

    #! Supported search logic capabilities
    /** This is a bitfield of capabilities; see @ref logic_capability_codes for more info
    */
    int search_logic_capabilities = LC_All;

    #! A hash of mapper key information
    /** @note
        - this is a static attribute; it is the same for all data providers of the same type
        - the \a handler keys are not returned as they are only valid at runtime
    */
    hash<string, hash<MapperRuntimeKeyInfo>> mapper_keys = Mapper::MapperKeyInfo;

    #! A markdown description of the data provider
    /** @note this is a dynamic attribute; it differs for different data providers of the same type
    */
    *string desc;

    #! A list of child data providers in this data provider
    /** @note this is a dynamic attribute; it differs for different data providers of the same type
    */
    *list<string> children;
}

#! The AbstractDataProvider class
public class AbstractDataProvider inherits Logger::LoggerWrapper {
    public {
        #! Generic expression implementations for data providers without native search functionality
        const GenericExpressionImplementations = {
            DP_OP_AND: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_OP_AND,
                    "name": "and (&&)",
                    "desc": "logical AND operation supporting logic short-circuiting",
                    "symbol": "&&",
                    "args": (DataProviderSignatureBoolType),
                    "varargs": True,
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    foreach auto arg in (exp.args) {
                        if (!AbstractDataProvider::evalGenericExpressionValue(rec, arg)) {
                            return False;
                        }
                    }
                    return True;
                },
            },
            DP_OP_OR: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_OP_OR,
                    "name": "or (||)",
                    "desc": "logical OR operation supporting logic short-circuiting",
                    "symbol": "||",
                    "args": (DataProviderSignatureBoolType),
                    "varargs": True,
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    foreach auto arg in (exp.args) {
                        if (AbstractDataProvider::evalGenericExpressionValue(rec, arg)) {
                            return True;
                        }
                    }
                    return False;
                },
            },
            DP_SEARCH_OP_EQ: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_EQ,
                    "name": "equals (=)",
                    "desc": "a value for equality comparisons; the type of the value should correspond to the field "
                        "type",
                    "symbol": "==",
                    "args": (DataProviderSignatureAnyType, DataProviderSignatureAnyType),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    auto arg0 = AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0]);
                    auto arg1 = AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]);
                    # for backwards compatibility, if the first argument is a field reference, and the second is a
                    # hash

                    return arg0 == arg1;
                },
            },
            DP_SEARCH_OP_NE: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_NE,
                    "name": "not equals (!=)",
                    "desc": "a value for not-equals comparisons; the type of the value should correspond to the "
                        "field type",
                    "symbol": "!=",
                    "args": (DataProviderSignatureAnyType, DataProviderSignatureAnyType),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    return AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0])
                        != AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]);
                },
            },
            DP_SEARCH_OP_LT: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_LT,
                    "name": "less than (<)",
                    "desc": "a value for less than comparisons; if the field value is less than the argument, then "
                        "the operation returns true; the type of the value should correspond to the field type",
                    "symbol": "<",
                    "args": (DataProviderSignatureAnyType, DataProviderSignatureAnyType),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    return AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0])
                        < AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]);
                },
            },
            DP_SEARCH_OP_LE: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_LE,
                    "name": "less than or equals (<=)",
                    "desc": "a value for less than or equals comparisons; if the field value is less than or equal "
                        "to the argument, then the operation returns true; the type of the value should correspond "
                        "to the field type",
                    "symbol": "<=",
                    "args": (DataProviderSignatureAnyType, DataProviderSignatureAnyType),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    return AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0])
                        <= AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]);
                },
            },
            DP_SEARCH_OP_GT: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_GT,
                    "name": "greater than (>)",
                    "desc": "a value for less than comparisons; if the field value is less than the argument, then "
                        "the operation returns true; the type of the value should correspond to the field type",
                    "symbol": ">",
                    "args": (DataProviderSignatureAnyType, DataProviderSignatureAnyType),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    return AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0])
                        > AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]);
                },
            },
            DP_SEARCH_OP_GE: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_GE,
                    "name": "greater than or equals (>=)",
                    "desc": "a value for greater than or equals comparisons; if the field value is greater than or "
                        "equal to the argument, then the operation returns true; the type of the value should "
                        "correspond to the field type",
                    "symbol": ">=",
                    "args": (DataProviderSignatureAnyType, DataProviderSignatureAnyType),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    return AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0])
                        >= AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]);
                },
            },
            DP_SEARCH_OP_BETWEEN: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_BETWEEN,
                    "name": "between",
                    "desc": "A list with two elements giving the lower and upper bounds of the field value; the list "
                        "element value types must be equal to the field's type",
                    "symbol": "between",
                    "args": (
                        DataProviderSignatureAnyType,
                        DataProviderSignatureAnyType,
                        DataProviderSignatureAnyType,
                    ),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    auto arg0 = AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0]);
                    return arg0 > AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1])
                        && arg0 < AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[2]);
                },
            },
            DP_SEARCH_OP_IN: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_IN,
                    "name": "in",
                    "desc": "A list giving possible values of the field;  if the field's value matches any of the "
                        "values in the list, then the operation returns true; element value types must be equal to "
                        "the field's type",
                    "symbol": "in",
                    "args": (DataProviderSignatureAnyType),
                    "return_type": AbstractDataProviderTypeMap."bool",
                    "varargs": True,
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    return inlist(AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0]),
                        AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]));
                },
            },
            DP_SEARCH_OP_NOT: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_NOT,
                    "name": "logical not (!)",
                    "desc": "This operator reverses the logcal value of the operator expression given as an argument",
                    "symbol": "!",
                    "args": (DataProviderSignatureBoolType,),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    return (!AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0]));
                },
            },
            DP_SEARCH_OP_REGEX: {
                "exp": <DataProviderExpressionInfo>{
                    "type": DET_Operator,
                    "label": DP_SEARCH_OP_REGEX,
                    "name": "regular expression match",
                    "desc": "regular expression operator; the first argument will be processed with the second as a "
                        "regular expression pattern",
                    "symbol": "regex",
                    "args": (
                        DataProviderSignatureStringType,
                        <DataProviderSignatureTypeInfo>{
                            "type": (new HashDataType())
                                .addField(new QoreDataField("pattern", "the regular expression pattern",
                                    AbstractDataProviderTypeMap."string"))
                                .addField(new QoreDataField("options", "regular expression options",
                                    AbstractDataProviderTypeMap."*int")),
                        },
                    ),
                    "return_type": AbstractDataProviderTypeMap."bool",
                },
                "impl": auto sub (hash<auto> rec, hash<DataProviderExpression> exp) {
                    auto arg1 = AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[1]);
                    return regex(AbstractDataProvider::evalGenericExpressionValue(rec, exp.args[0]).toString(),
                        arg1.pattern, arg1.options);
                },
            },
        };

        #! Generic search operator expressions
        /** The following are generic search operator expressions implemented for all data providers that do not
            provide native search functionality

            @since DataProvider 2.3
        */
        const GenericExpressions = map {$1.key: $1.value.exp}, GenericExpressionImplementations.pairIterator();

        #! A list of members names of the DataProviderSummaryInfoKeys hashdecl
        const DataProviderSummaryInfoKeys = map $1.getName(),
            TypedHash::forName("DataProviderSummaryInfo").getMembers();
    }

    private {
        #! flag if callbacks are locked
        static bool callbacks_locked = False;

        #! static callback the returns a bool if the value needs dynamic resolution
        static code cb_value_needs_resolution;

        #! static callback for dynamic value resolution
        static code cb_resolve_value;

        #! Generic record search options; see details below
        /** Each record-based data provider should support the following search options:
            - \c columns: a list of one or more field names; allows a subset of fields to be output in searches
            - \c limit: the maximum number of records to return
            - \c offset: the offset number in records to return
        */
        const GenericRecordSearchOptions = {
            "columns": <DataProviderOptionInfo>{
                "type": AbstractDataProviderTypeMap."any",
                "desc": "allows a subset of fields to be output in searches",
            },
            "limit": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(IntType),
                "desc": "the maximum number of records to return",
            },
            "offset": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(IntType),
                "desc": "the offset number in records to return",
            },
        };
    }

    #! Creates the data provider
    constructor() {
    }

    #! Creates the data provider with the given Logger
    constructor(LoggerInterface logger) : LoggerWrapper(logger) {
    }

    #! Returns static provider information as data; no objects are returned
    /** @param with_type_info if @ref True then more type information will be returned in the response

        @note the \c name and \c children attributes are not returned as they are dynamic attributes
    */
    hash<auto> getInfoAsData(*bool with_type_info) {
        return DataProvider::getInfoAsData(getInfo(), with_type_info);
    }

    #! Returns data provider info
    hash<DataProviderInfo> getInfo() {
        hash<DataProviderInfo> rv();
        rv += getStaticInfoImpl() + {
            "children": getChildProviderNames(),
        };
        if (!rv.name) {
            rv.name = getName();
        }
        if (!rv.desc) {
            rv.desc = getDesc() ?? "no description available";
        }
        if (!exists rv.has_record) {
            rv.has_record = rv.supports_read;
        }
        if (!rv.events && rv.supports_observable) {
            rv.events = getEventTypes();
        }
        if (!rv.messages && rv.supports_messages != MSG_None) {
            rv.messages = getMessageTypes();
        }
        return rv;
    }

    #! Return data provider summary info
    hash<DataProviderSummaryInfo> getSummaryInfo() {
        hash<DataProviderInfo> info = getStaticInfoImpl();
        if (!info.name) {
            info.name = getName();
        }
        if (!info.desc) {
            info.desc = getDesc() ?? "no description available";
        }
        return cast<hash<DataProviderSummaryInfo>>(info{DataProviderSummaryInfoKeys});
    }

    #! Returns the data provider description
    *string getDesc() {
        return getStaticInfoImpl().desc;
    }

    #! Creates the given record in the data provider
    /** @param rec a hash representing a single input record
        @param create_options the create options; will be processed by validateCreateOptions()

        @return the data written to the data provider

        @throw INVALID-OPERATION the data provider does not support record creation
        @throw CREATE-RECORD-ERROR the data provider raised an exception in the record creation operation
        @throw DUPLICATE-RECORD this exception should be thrown if the provider fails due to an attempt to create a
        duplicate record
    */
    *hash<auto> createRecord(hash<auto> rec, *hash<auto> create_options) {
        checkCreate();
        on_error {
            if ($1.err != "DUPLICATE-RECORD" && $1.err != "CREATE-RECORD-ERROR") {
                rethrow "CREATE-RECORD-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
            }
        }
        return createRecordImpl(rec, validateCreateOptions(create_options));
    }

    #! Upserts the given record in the data provider
    /** @param rec a hash representing a single input record
        @param upsert_options the upsert options; will be processed by validateUpsertOptions()

        @return see @ref db_provider_upsert_results for possible values

        @throw INVALID-OPERATION the data provider does not support upsert operations
    */
    string upsertRecord(hash<auto> rec, *hash<auto> upsert_options) {
        checkUpsert();
        return upsertRecordImpl(rec, validateUpsertOptions(upsert_options));
    }

    #! Returns the first record matching the search options
    /** @param where_cond the search criteria; will be processed by processSearchParameters()
        @param search_options the search options; will be processed by validateSearchOptions()

        @throw INVALID-OPERATION the data provider does not support reading

        @since DataProvider 2.3
    */
    *hash<auto> searchFirstRecord(hash<DataProviderExpression> where_cond, *hash<auto> search_options) {
        checkRead();
        search_options = validateSearchOptions(search_options);
        return searchFirstRecordImpl(processSearchParameters(where_cond, search_options), search_options);
    }

    #! Returns the first record matching the search options
    /** @param where_cond the search criteria; will be processed by processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @throw INVALID-OPERATION the data provider does not support reading
    */
    *hash<auto> searchFirstRecord(hash<auto> where_cond, *hash<auto> search_options) {
        checkRead();
        search_options = validateSearchOptions(search_options);
        return searchFirstRecordImpl(getSearchExpression(where_cond, search_options), search_options);
    }

    #! Returns a single record matching the search options
    /** @param where_cond the search criteria; will be processed by processSearchParameters()
        @param search_options the search options; will be processed by validateSearchOptions()

        @throw INVALID-OPERATION the data provider does not support reading
        @throw MULTIPLE-RECORDS-ERROR multiple records found

        @since DataProvider 2.3
    */
    *hash<auto> searchSingleRecord(hash<DataProviderExpression> where_cond, *hash<auto> search_options) {
        checkRead();
        search_options = validateSearchOptions(search_options);
        return searchSingleRecordImpl(processSearchParameters(where_cond, search_options), search_options);
    }

    #! Returns a single record matching the search options
    /** @param where_cond the search criteria; will be processed by processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @throw INVALID-OPERATION the data provider does not support reading
        @throw MULTIPLE-RECORDS-ERROR multiple records found
    */
    *hash<auto> searchSingleRecord(hash<auto> where_cond, *hash<auto> search_options) {
        checkRead();
        search_options = validateSearchOptions(search_options);
        return searchSingleRecordImpl(getSearchExpression(where_cond, search_options), search_options);
    }

    #! Returns an iterator iterating all records
    /** @param search_options the search options; will be processed by validateSearchOptions()

        @throw INVALID-OPERATION the data provider does not support reading
    */
    AbstractDataProviderRecordIterator getRecordIterator(*hash<auto> search_options) {
        checkRead();
        return searchRecords(NOTHING, validateSearchOptions(search_options));
    }

    #! Returns an iterator iterating all records with the bulk read API
    /** @param block_size the number of records in a read block; must be a positive number
        @param search_options the search options; will be processed by validateSearchOptions()

        @return a bulk record interface object that will return the records in bulk format

        @throw INVALID-BLOCK-SIZE the block size must be a positive number
        @throw INVALID-OPERATION the data provider does not support reading
    */
    AbstractDataProviderBulkRecordInterface getBulkRecordInterface(int block_size = 1000,
            *hash<auto> search_options) {
        checkRead();
        return searchRecordsBulk(block_size, NOTHING, validateSearchOptions(search_options));
    }

    #! Returns an iterator for zero or more records matching the search options
    /** @param block_size the number of records in a read block; must be a positive number
        @param where_cond the search criteria; will be processed by processSearchParameters()
        @param search_options the search options; will be processed by validateSearchOptions()

        @return a bulk record interface object that will return the records in bulk format

        @throw INVALID-BLOCK-SIZE the block size must be a positive number
        @throw INVALID-OPERATION the data provider does not support reading

        @since DataProvider 2.3
    */
    AbstractDataProviderBulkRecordInterface searchRecordsBulk(int block_size = 1000,
            *hash<DataProviderExpression> where_cond, *hash<auto> search_options) {
        checkRead();
        if (block_size <= 0) {
            throw "INVALID-BLOCK-SIZE", sprintf("block_size %d is invalid; the block size mst be a positive number",
                block_size);
        }
        search_options = validateSearchOptions(search_options);
        return searchRecordsBulkImpl(block_size, processSearchParameters(where_cond, search_options),
            search_options);
    }

    #! Returns an iterator for zero or more records matching the search options
    /** @param block_size the number of records in a read block; must be a positive number
        @param where_cond the search criteria; will be processed by processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @return a bulk record interface object that will return the records in bulk format

        @throw INVALID-BLOCK-SIZE the block size must be a positive number
        @throw INVALID-OPERATION the data provider does not support reading
    */
    AbstractDataProviderBulkRecordInterface searchRecordsBulk(int block_size = 1000, *hash<auto> where_cond,
            *hash<auto> search_options) {
        checkRead();
        if (block_size <= 0) {
            throw "INVALID-BLOCK-SIZE", sprintf("block_size %d is invalid; the block size mst be a positive number",
                block_size);
        }
        search_options = validateSearchOptions(search_options);
        return searchRecordsBulkImpl(block_size, getSearchExpression(where_cond, search_options), search_options);
    }

    #! Returns an iterator for zero or more records matching the search options
    /** @param where_cond the search criteria; will be processed by processSearchParameters()
        @param search_options the search options; will be processed by validateSearchOptions()

        @throw INVALID-OPERATION the data provider does not support reading

        @since DataProvider 2.3
    */
    AbstractDataProviderRecordIterator searchRecords(*hash<DataProviderExpression> where_cond,
            *hash<auto> search_options) {
        checkRead();
        search_options = validateSearchOptions(search_options);
        return searchRecordsImpl(processSearchParameters(where_cond, search_options), search_options);
    }

    #! Returns an iterator for zero or more records matching the search options
    /** @param where_cond the search criteria; will be processed by processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @throw INVALID-OPERATION the data provider does not support reading
    */
    AbstractDataProviderRecordIterator searchRecords(*hash<auto> where_cond, *hash<auto> search_options) {
        checkRead();
        search_options = validateSearchOptions(search_options);
        return searchRecordsImpl(getSearchExpression(where_cond, search_options), search_options);
    }

    #! Returns an iterator for zero or more records matching the search options according to an API request
    /** @param req the request to serialize and make according to the request type
        @param where_cond the search criteria; will be processed by processFieldValues()
        @param search_options the search options after processing by validateSearchOptions()

        This will execute the request and perform a default search on any record(s) returned

        @throw INVALID-OPERATION the data provider does not support reading records or the request / response API

        @since DataProvider 2.3
    */
    AbstractDataProviderRecordIterator requestSearchRecords(auto req, *hash<DataProviderExpression> where_cond,
            *hash<auto> search_options) {
        checkRead();
        checkRequest();
        return requestSearchRecordsImpl(req, processSearchParameters(where_cond, search_options), search_options);
    }

    #! Returns an iterator for zero or more records matching the search options according to an API request
    /** @param req the request to serialize and make according to the request type
        @param where_cond the search criteria; will be processed by processFieldValues()
        @param search_options the search options after processing by validateSearchOptions()

        This will execute the request and perform a default search on any record(s) returned

        @throw INVALID-OPERATION the data provider does not support reading records or the request / response API
    */
    AbstractDataProviderRecordIterator requestSearchRecords(auto req, *hash<auto> where_cond,
            *hash<auto> search_options) {
        checkRead();
        checkRequest();
        return requestSearchRecordsImpl(req, getSearchExpression(where_cond, search_options), search_options);
    }

    #! Updates a single record matching the search options
    /** @param set the hash of field data to set; will be processed by processFieldValues()
        @param where_cond the search criteria; will be processed by processSearchParameters()
        @param search_options the search options; will be processed by validateSearchOptions()

        @returns @ref True if the record was updated, @ref False if not (no record found)

        @throw INVALID-OPERATION the data provider does not support record updating
        @throw UPDATE-RECORDS-ERROR error updating records

        @since DataProvider 2.3
    */
    bool updateSingleRecord(hash<auto> set, hash<DataProviderExpression> where_cond, *hash<auto> search_options) {
        checkUpdate();
        search_options = validateSearchOptions(search_options);
        on_error {
            if ($1.err != "DUPLICATE-RECORD" && $1.err != "UPDATE-RECORDS-ERROR") {
                rethrow "UPDATE-RECORDS-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
            }
        }
        return updateSingleRecordImpl(processFieldValues(set, search_options),
            processSearchParameters(where_cond, search_options), search_options);
    }

    #! Updates a single record matching the search options
    /** @param set the hash of field data to set; will be processed by processFieldValues()
        @param where_cond the search criteria; will be processed by processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @returns @ref True if the record was updated, @ref False if not (no record found)

        @throw INVALID-OPERATION the data provider does not support record updating
        @throw UPDATE-RECORDS-ERROR error updating records
    */
    bool updateSingleRecord(hash<auto> set, hash<auto> where_cond, *hash<auto> search_options) {
        checkUpdate();
        search_options = validateSearchOptions(search_options);
        on_error {
            if ($1.err != "DUPLICATE-RECORD" && $1.err != "UPDATE-RECORDS-ERROR") {
                rethrow "UPDATE-RECORDS-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
            }
        }
        return updateSingleRecordImpl(processFieldValues(set, search_options),
            getSearchExpression(where_cond, search_options), search_options);
    }

    #! Updates zero or more records matching the search options
    /** @param set the hash of field data to set
        @param where_cond a hash for identifying the record(s) to be updated; will be processed by
        processSearchParameters()
        @param search_options the search options; will be processed by validateSearchOptions()

        @return the number of records updated

        @throw INVALID-OPERATION the data provider does not support record updating
        @throw UPDATE-RECORDS-ERROR error updating records

        @since DataProvider 2.3
    */
    int updateRecords(hash<auto> set, *hash<DataProviderExpression> where_cond, *hash<auto> search_options) {
        checkUpdate();
        search_options = validateSearchOptions(search_options);
        on_error {
            if ($1.err != "DUPLICATE-RECORD" && $1.err != "UPDATE-RECORDS-ERROR") {
                rethrow "UPDATE-RECORDS-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
            }
        }
        return updateRecordsImpl(processFieldValues(set, search_options),
            processSearchParameters(where_cond, search_options), search_options);
    }

    #! Updates zero or more records matching the search options
    /** @param set the hash of field data to set
        @param where_cond a hash for identifying the record(s) to be updated; will be processed by
        processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @return the number of records updated

        @throw INVALID-OPERATION the data provider does not support record updating
        @throw UPDATE-RECORDS-ERROR error updating records
    */
    int updateRecords(hash<auto> set, *hash<auto> where_cond, *hash<auto> search_options) {
        checkUpdate();
        search_options = validateSearchOptions(search_options);
        on_error {
            if ($1.err != "DUPLICATE-RECORD" && $1.err != "UPDATE-RECORDS-ERROR") {
                rethrow "UPDATE-RECORDS-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
            }
        }
        return updateRecordsImpl(processFieldValues(set, search_options),
            getSearchExpression(where_cond, search_options), search_options);
    }

    #! Deletes zero or more records
    /**
        @param where_cond a hash for identifying the record(s) to be deleted; will be processed by
        processSearchParameters()
        @param search_options the search options; will be processed by validateSearchOptions()

        @return the number of records deleted

        @throw INVALID-OPERATION the data provider does not support record deletion
        @throw DELETE-RECORDS-ERROR error deleting records

        @since DataProvider 2.3
    */
    int deleteRecords(*hash<DataProviderExpression> where_cond, *hash<auto> search_options) {
        checkDelete();
        search_options = validateSearchOptions(search_options);
        on_error {
            if ($1.err != "DELETE-RECORDS-ERROR") {
                rethrow "DELETE-RECORDS-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
            }
        }
        return deleteRecordsImpl(processSearchParameters(where_cond, search_options), search_options);
    }

    #! Deletes zero or more records
    /**
        @param where_cond a hash for identifying the record(s) to be deleted; will be processed by
        processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @return the number of records deleted

        @throw INVALID-OPERATION the data provider does not support record deletion
        @throw DELETE-RECORDS-ERROR error deleting records
    */
    int deleteRecords(*hash<auto> where_cond, *hash<auto> search_options) {
        checkDelete();
        search_options = validateSearchOptions(search_options);
        on_error {
            if ($1.err != "DELETE-RECORDS-ERROR") {
                rethrow "DELETE-RECORDS-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
            }
        }
        return deleteRecordsImpl(getSearchExpression(where_cond, search_options), search_options);
    }

    #! Makes a request and returns the response
    /** @param req the request to serialize and make according to the request type
        @param request_options the request options; will be processed by validateRequestOptions()

        @return the response to the request

        @throw INVALID-OPERATION the data provider does not support the request API
        @throw INVALID-REQUEST the request argument provided do not match the expected request type
    */
    auto doRequest(auto req, *hash<auto> request_options) {
        checkRequest();
        *AbstractDataProviderType request_type = getRequestType();
        if (request_type) {
            try {
                req = request_type.acceptsValue(req);
            } catch (hash<ExceptionInfo> ex) {
                rethrow "INVALID-REQUEST", sprintf("invalid request argument for request %y: %s: %s", getName(),
                    ex.err, ex.desc);
            }
        }
        return doRequestImpl(req, validateRequestOptions(request_options));
    }

    #! Creates a new child data provider and returns it after adding as a child
    /** @param name the name of the new child data provider
        @param fields the fields for the new child data provider; must have at least one field
        @param child_create_options the options for creating the new child data provider

        @return the new child data provider

        @throw INVALID-OPERATION thrown if the data provider does not support this API
        @throw CREATE-CHILD-PROVIDER-ERROR error creating the child provider

        @since DataProvider 2.0
    */
    AbstractDataProvider createChildProvider(string name, hash<string, AbstractDataField> fields,
            *hash<auto> child_create_options) {
        checkCreateChild();
        child_create_options = validateChildCreateOptions(child_create_options);
        if (!fields) {
            throw "CREATE-CHILD-PROVIDER-ERROR", "at least one field must be present in the field hash";
        }
        on_error if ($1.err != "CREATE-CHILD-PROVIDER-ERROR") {
            rethrow "CREATE-CHILD-PROVIDER-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
        }
        AbstractDataProvider child = createChildProviderImpl(name, fields, child_create_options);
        if (logger && !child.logger) {
            child.setLogger(logger);
        }
        return child;
    }

    #! Deletes a child data provider
    /** @param name the name of the new child data provider
        @param child_delete_options the options for deleting a child data provider

        @throw INVALID-OPERATION thrown if the data provider does not support this API
        @throw DELETE-CHILD-PROVIDER-ERROR error creating the child provider

        @since DataProvider 2.0
    */
    deleteChildProvider(string name, *hash<auto> child_delete_options) {
        checkDeleteChild();
        child_delete_options = validateChildDeleteOptions(child_delete_options);
        on_error if ($1.err != "DELETE-CHILD-PROVIDER-ERROR") {
            rethrow "DELETE-CHILD-PROVIDER-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
        }
        return deleteChildProviderImpl(name, child_delete_options);
    }

    #! Creates a new field
    /**
        @param field the field to add
        @param field_add_options the options for adding the new field

        @throw INVALID-OPERATION thrown if the data provider does not support this API
        @throw ADD-FIELD-ERROR error adding the field

        @since DataProvider 2.0
    */
    addField(AbstractDataField field, *hash<auto> field_add_options) {
        checkAddField();
        field_add_options = validateFieldAddOptions(field_add_options);
        on_error if ($1.err != "ADD-FIELD-ERROR") {
            rethrow "ADD-FIELD-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
        }
        addFieldImpl(field, field_add_options);
    }

    #! Updates an existing field
    /** @param name the name of the current field to update
        @param field the new definition of the field
        @param field_update_options the options for adding the new field

        @throw INVALID-OPERATION thrown if the data provider does not support this API
        @throw UPDATE-FIELD-ERROR error adding the field

        @since DataProvider 2.0
    */
    updateField(string name, AbstractDataField field, *hash<auto> field_update_options) {
        checkUpdateField();
        field_update_options = validateFieldUpdateOptions(field_update_options);
        on_error if ($1.err != "UPDATE-FIELD-ERROR") {
            rethrow "UPDATE-FIELD-ERROR", sprintf("%s: %s", $1.err, $1.desc), $1.arg;
        }
        return updateFieldImpl(name, field, field_update_options);
    }

    #! Deletes an existing field
    /** @param name the name of the field to delete
        @param field_delete_options the options for deleting the new field; processed by validateFieldDeleteOptions()

        @throw INVALID-OPERATION thrown if the data provider does not support this API
        @throw DELETE-FIELD-ERROR error adding the field

        @since DataProvider 2.0
    */
    deleteField(string name, *hash<auto> field_delete_options) {
        checkDeleteField();
        field_delete_options = validateFieldDeleteOptions(field_delete_options);
        on_error if ($1.err != "DELETE-FIELD-ERROR") {
            rethrow "DELETE-FIELD-ERROR", sprintf("%s: %s", $1.err, $1.desc);
        }
        return deleteFieldImpl(name, field_delete_options);
    }

    #! Sends a message from message-capable data providers
    /**
        @param message_id the message ID to send; must correspond to a valid message ID
        @param msg the message to send
        @param send_message_options the options for sending the message; processed by validateSendMessageOptions()

        @throw INVALID-OPERATION thrown if the data provider does not support this API
        @throw INVALID-MESSAGE the message_id is not valid
        @throw SEND-MESSAGE-ERROR error sending the message

        @since DataProvider 2.5
    */
    sendMessage(string message_id, auto msg, *hash<auto> send_message_options) {
        checkMessages();
        send_message_options = validateSendMessageOptions(send_message_options);
        hash<DataProviderMessageInfo> message_info = getMessageInfoImpl(message_id);
        on_error if ($1.err != "SEND-MESSAGE-ERROR") {
            rethrow "SEND-MESSAGE-ERROR", sprintf("%s: (message ID %y) %s", $1.err, message_id, $1.desc);
        }
        msg = message_info.type.acceptsValue(msg);
        sendMessageImpl(message_id, msg, send_message_options);
    }

    #! Returns the schema supporting this data provider
    /** @return the schema supporting this data provider

        @throw INVALID-OPERATION thrown if the data provider does not support this API

        @since DataProvider 2.2
    */
    object getSchemaObject() {
        checkSchema();
        return getSchemaObjectImpl();
    }

    #! Returns the description of a successful request message, if any
    /** @return the request type for this provider

        @throw INVALID-OPERATION the data provider does not support the request API
    */
    *AbstractDataProviderType getRequestType() {
        checkRequest();
        return getRequestTypeImpl();
    }

    #! Returns the description of a response message, if this object represents a response message
    /** @return the response type for this response message

        @throw INVALID-OPERATION the data provider does not support the request API
    */
    *AbstractDataProviderType getResponseType() {
        checkRequest();
        return getResponseTypeImpl();
    }

    #! Returns a hash of error responses, if any
    /** @return a hash of error responses, if any, keyed by response code or string

        @throw INVALID-OPERATION the data provider does not support the request API
    */
    *hash<string, AbstractDataProviderType> getErrorResponseTypes() {
        checkRequest();
        return getErrorResponseTypesImpl();
    }

    #! Returns the type for the given error code
    /** @param error_code the error code for the response; must be a known error code, or an \c UNKNOWN-ERROR-RESPONSE
        exception is thrown
        @return the type for the given error code

        @throw INVALID-OPERATION the data provider does not support the request API
        @throw UNKNOWN-ERROR-RESPONSE the error response given is not known
    */
    AbstractDataProviderType getErrorResponseType(string error_code) {
        checkRequest();
        return getErrorResponseTypeImpl(error_code);
    }

    #! Returns the description of an observable event, if any
    /** @param event_id the event ID of the event

        @return the observable event type for this provider

        @throw INVALID-OPERATION the data provider does not support the observer pattern / event API
        @throw INVALID-EVENT the event_id is not valid

        @since DataProvider 2.5
    */
    hash<DataProviderMessageInfo> getEventInfo(string event_id) {
        checkObservable();
        return getEventInfoImpl(event_id);
    }

    #! Returns a hash of all supported event types
    /** @return a hash of all supported event types

        @throw INVALID-OPERATION the data provider does not support the observer pattern / event API

        @since DataProvider 2.5
    */
    hash<string, hash<DataProviderMessageInfo>> getEventTypes() {
        checkObservable();
        return getEventTypesImpl();
    }

    #! Returns the description of an outbound message, if any
    /** @param message_id the ID of the message

        @return the message type for this provider

        @throw INVALID-OPERATION the data provider does not support sending messages
        @throw INVALID-MESSAGE the message_id is not valid

        @since DataProvider 2.5
    */
    hash<DataProviderMessageInfo> getMessageInfo(string message_id) {
        checkMessages();
        return getMessageInfoImpl(message_id);
    }

    #! Returns a hash of all supported outbound messages
    /** @return a hash of all supported outbound messages

        @throw INVALID-OPERATION the data provider does not support sending messages

        @since DataProvider 2.5
    */
    hash<string, hash<DataProviderMessageInfo>> getMessageTypes() {
        checkMessages();
        return getMessageTypesImpl();
    }

    #! Return data provider summary info
    /** @note This creates an AbstractDataProvider object for each child; for cases when this is expensive, this
        method must be overridden / reimplemented in the child class for performance reasons
    */
    *list<hash<DataProviderSummaryInfo>> getChildProviderSummaryInfo() {
        *list<hash<DataProviderSummaryInfo>> rv;
        foreach string name in (getChildProviderNames()) {
            try {
                rv += getChildProviderEx(name).getSummaryInfo();
            } catch (hash<ExceptionInfo> ex) {
                # ignore exception
            }
        }
        return rv;
    }

    #! Returns a list of child data provider names, if any
    /** @return a list of child data provider names, if any
    */
    *list<string> getChildProviderNames() {
        return getChildProviderNamesImpl();
    }

    #! Returns the given child provider or @ref nothing if the given child is unknown
    /** @return the given child provider or @ref nothing if the given child is unknown

        @throw INVALID-CHILD-PROVIDER unknown child provider

        @see getChildProviderEx()
    */
    *AbstractDataProvider getChildProvider(string name) {
        *AbstractDataProvider child = getChildProviderImpl(name);
        if (logger && !child.logger) {
            child.setLogger(logger);
        }
        return child;
    }

    #! Returns the given child provider or throws an exception if the given child is unknown
    /** @return the given child provider or throws an exception if the given child is unknown

        @throw INVALID-CHILD-PROVIDER unknown child provider

        @see getChildProvider()
    */
    AbstractDataProvider getChildProviderEx(string name) {
        *AbstractDataProvider provider = getChildProviderImpl(name);
        if (!provider) {
            throw "INVALID-CHILD-PROVIDER", sprintf("provider %y is unknown; known children of %y: %y", name,
                getName(), getChildProviderNames());
        }
        if (logger && !provider.logger) {
            provider.setLogger(logger);
        }
        return provider;
    }

    #! Returns the given child provider from a \c "/" separated path string
    /** throws an exception with unknown path elements

        @param path a string giving a path to the target provider where child elements are separated by \c "/" characters

        @return the target child provider; throws an exception if any element in the path is unknown

        @throw INVALID-CHILD-PROVIDER unknown child provider
    */
    AbstractDataProvider getChildProviderPath(string path) {
        AbstractDataProvider provider = self;
        map provider = provider.getChildProviderEx($1), path.split("/");
        if (logger && !provider.logger) {
            provider.setLogger(logger);
        }
        return provider;
    }

    #! Returns @ref True if the data provider requires transaction management
    /** @return @ref True if the data provider requires transaction management, in which case commit() or rollback()
        must be called to flush or discard data written to the data provider
    */
    bool requiresTransactionManagement() {
        return getStaticInfoImpl().transaction_management;
    }

    #! Begins a transaction with a data provider
    /** Has no effect if the data provider does not require transaction management

        @see:
        - commit()
        - rollback()

        @since DataProvider 1.2.2
    */
    beginTransaction() {
    }

    #! Commits data written to the data provider
    /** Has no effect if the data provider does not support transaction management

        @see:
        - beginTransaction()
        - rollback()
    */
    commit() {
    }

    #! Rolls back data written to the data provider
    /** Has no effect if the data provider does not support transaction management

        @see:
        - beginTransaction()
        - commit()
    */
    rollback() {
    }

    #! Returns a bulk insert operation object for the data provider
    /** @return a bulk insert operation object for the data provider

        @throw INVALID-OPERATION the data provider does not support create operations
    */
    AbstractDataProviderBulkOperation getBulkInserter() {
        checkCreate();
        return new DefaultBulkInserter(self);
    }

    #! Returns a bulk upsert operation object for the data provider
    /** @return a bulk upsert operation object for the data provider

        @throw INVALID-OPERATION the data provider does not support upsert operations
    */
    AbstractDataProviderBulkOperation getBulkUpserter() {
        checkUpsert();
        return new DefaultBulkUpserter(self);
    }

    #! Returns custom data mapper runtime keys
    /** @return custom data mapper runtime keys

        This base method returns @ref nothing; reimplment in child classes to return a value
    */
    *hash<string, hash<MapperRuntimeKeyInfo>> getMapperRuntimeKeys() {
        # this method intentionally left empty
    }

    #! Ensures that the data provider supports read operations
    /**
        @throw INVALID-OPERATION the data provider does not support read operations
    */
    checkRead() {
        if (!supportsRead()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support reading records",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports record creation
    /**
        @throw INVALID-OPERATION the data provider does not support record creation
    */
    checkCreate() {
        if (!supportsCreate()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support record creation",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports record upserts
    /**
        @throw INVALID-OPERATION the data provider does not support upsert operations
    */
    checkUpsert() {
        if (!supportsUpsert()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support record upserts",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports record upserts
    /**
        @throw INVALID-OPERATION the data provider does not support record updating
    */
    checkUpdate() {
        if (!supportsUpdate()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support record updating",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports record deletion
    /**
        @throw INVALID-OPERATION the data provider does not support record deletion
    */
    checkDelete() {
        if (!supportsDelete()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support record deletion",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports the request API
    /**
        @throw INVALID-OPERATION the data provider does not support the request API
    */
    checkRequest() {
        if (!supportsRequest()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support the request API",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports creating children
    /**
        @throw INVALID-OPERATION the data provider does not support the child creation API

        @since DataProvider 2.0
    */
    checkCreateChild() {
        if (!supportsCreateChild()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support creating children",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports deleting children
    /**
        @throw INVALID-OPERATION the data provider does not support the child deletion API

        @since DataProvider 2.0
    */
    checkDeleteChild() {
        if (!supportsDeleteChild()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support deleting children",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports adding fields
    /**
        @throw INVALID-OPERATION the data provider does not support the add field API

        @since DataProvider 2.0
    */
    checkAddField() {
        if (!supportsAddField()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support adding fields",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports updating fields
    /**
        @throw INVALID-OPERATION the data provider does not support the update field API

        @since DataProvider 2.0
    */
    checkUpdateField() {
        if (!supportsUpdateField()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support updating fields",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports deleting fields
    /**
        @throw INVALID-OPERATION the data provider does not support the delete field API

        @since DataProvider 2.0
    */
    checkDeleteField() {
        if (!supportsDeleteField()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support deleting fields",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports a schema
    /**
        @throw INVALID-OPERATION the data provider does not support a schema

        @since DataProvider 2.2
    */
    checkSchema() {
        if (!supportsSchema()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support schemas",
                getName(), self.className());
        }
    }

    #! Ensures that the data provider supports advanced search expressions
    /**
        @throw INVALID-OPERATION the data provider does not support advanced search expressions

        @since DataProvider 2.2
    */
    checkSearchExpressions() {
        if (!supportsSearchExpressions()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support advanced search "
                "expressions", getName(), self.className());
        }
    }

    #! Ensures that the data provider supports observer pattern / event API
    /**
        @throw INVALID-OPERATION the data provider does not support the observer pattern / event API

        @since DataProvider 2.5
    */
    checkObservable() {
        if (!supportsObservable()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support the observer pattern / "
                "event API", getName(), self.className());
        }
    }

    #! Ensures that the data provider supports sending messages
    /**
        @throw INVALID-OPERATION the data provider does not support sending messages

        @since DataProvider 2.5
    */
    checkMessages() {
        if (!supportsMessages()) {
            throw "INVALID-OPERATION", sprintf("data provider %y (type %y) does not support sending messages",
                getName(), self.className());
        }
    }

    #! Returns a search expression for a standard search hash
    /** @param where_cond the search criteria; if the data provider supports advanced search, and the value is an
        expression, it will be processed by processSearchParameters(), otherwise by processFieldValues()
        @param search_options the search options; will be processed by validateSearchOptions()

        @return a search expression corresponding to \a where_cond

        @since DataProvider 2.3
    */
    *hash<auto> getSearchExpression(*hash<auto> where_cond, *hash<auto> search_options) {
        if (where_cond instanceof hash<DataProviderExpression>) {
            return processSearchParameters(where_cond, search_options);
        }
        if (!where_cond) {
            return where_cond;
        }
        if (!supportsSearchExpressions()) {
            return processFieldValues(where_cond, search_options);
        }
        where_cond = processFieldValues(where_cond, search_options);
        hash<DataProviderInfo> info = getInfo();
        hash<DataProviderExpressionInfo> expinfo = getExpression(ER_Search, DP_OP_AND, info.expressions);
        hash<DataProviderExpression> rv = <DataProviderExpression>{
            "exp": DP_OP_AND,
            "args": map getSimpleArgumentIntern(ER_Search, info, expinfo, $1.key, $1.value),
                where_cond.pairIterator(),
        };
        return rv;
    }

    #! Returns an argument for an expression
    /**
        @since DataProvider 2.3
    */
    private hash<DataProviderExpression> getSimpleArgumentIntern(int role, hash<DataProviderInfo> info,
            hash<DataProviderExpressionInfo> expinfo, string key, auto value) {
        # remove any unique suffix from key to get field name
        key =~ s/:.*//g;
        hash<DataProviderExpression> rv;
        if (value.typeCode() == NT_HASH && value.op && value.hasKey("arg")) {
            hash<DataProviderExpressionInfo> expinfo = getExpression(role, value.op, info.expressions);
            rv = <DataProviderExpression>{
                "exp": expinfo.label,
                "args": value.arg.op && value.arg.arg
                    ? <DataProviderExpression>{
                        "exp": getExpression(role, value.arg.op, info.expressions).label,
                        "args": (<DataProviderFieldReference>{"field": key},) + value.arg.arg,
                    }
                    : (<DataProviderFieldReference>{"field": key}, ) + value.arg,
            };
        } else {
            rv = getEqualityComparisonExpression(role, info, key, value);
        }
        int caps = role == ER_Search ? info.search_logic_capabilities : 0;
        verifyExpression(role, info.expressions, caps, AbstractDataProviderTypeMap."bool", rv);
        return rv;
    }

    #! Returns an equality comparison expression
    /** Must be reimplemented by data provider implementations that support expressions and use a symbol different
        from '=' for the comparison operator

        @since DataProvider 2.3
    */
    private hash<DataProviderExpression> getEqualityComparisonExpression(int role, hash<DataProviderInfo> info,
            string key, auto value) {
        if (!info.expressions."=") {
            throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("data provider %s (type %y) provides no '=' operator; "
                "operators provided: %y", getName(), self.className(), keys info.expressions);
        }
        return <DataProviderExpression>{
            "exp": "=",
            "args": (<DataProviderFieldReference>{"field": key},) + value,
        };
    }

    #! Returns an expression definition for an expression code or throws an exception
    /**
        @since DataProvider 2.3
    */
    static private hash<DataProviderExpressionInfo> getExpression(int role, string exp,
            *hash<string, hash<DataProviderExpressionInfo>> expmap) {
        *hash<DataProviderExpressionInfo> expinfo = expmap{exp};
        if (!expinfo) {
            throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("expression %y is invalid; known expressions: %y", exp,
                keys expmap);
        }
        if (!(expinfo.role & role)) {
            throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("expression %y is invalid role %y", exp,
                RoleCodeMap{role});
        }
        return expinfo;
    }

    #! Ensures that no callbacks can be set for dynamic URI resolution
    /** @return False if this call was ignored because callbacks are locked, True if the call was successful

        @note after this call, callbacks cannot be changed
    */
    static bool setDynamicValueCallbacks() {
        if (!AbstractDataProvider::checkCallbacks()) {
            return !exists cb_resolve_value;
        }
        return True;
    }

    #! Evaluates the given expression with the generic internal implementation and returns the result
    /**
        @since DataProvider 2.3
    */
    static auto evalGenericExpressionValue(hash<auto> rec, auto val) {
        if (val instanceof hash<DataProviderFieldReference>) {
            # allow for dotted references to be respected; allow escaping of dots
            list<string> f = map regex_subst($1, "\\\\.", ".", RE_Global), val.field.splitRegex("(?<!\\\\)\\.");
            auto rv = rec;
            foreach string field in (f) {
                if (rv.typeCode() != NT_HASH && rv.typeCode() != NT_OBJECT) {
                    # only throw exceptions on the top-level key
                    if (!$#) {
                        throw "INVALID-OPERATION", sprintf("expression references invalid key %y of a value of type "
                            "%y in field reference %y", field, rv.fullType(), val.field);
                    }
                    # otherwise stop searching; no value will be returned
                    break;
                }
                auto new_rv = rv{field};
                # only throw exceptions on the top-level key
                if (!$# && !exists new_rv && !rv.hasKey(field)) {
                    throw "INVALID-OPERATION", sprintf("expression references invalid key %y in field reference "
                        "%y; valid keys: %y", field, val.field, keys rv);
                }
                rv = new_rv;
            }
            return rv;
        }
        if (val instanceof hash<DataProviderExpression>) {
            return AbstractDataProvider::evalGenericExpression(rec, val);
        }
        return val;
    }

    #! Evaluates the given expression with the generic internal implementation and returns the result
    /**
        @since DataProvider 2.3
    */
    static auto evalGenericExpression(hash<auto> rec, hash<DataProviderExpression> exp) {
        *hash<auto> expinfo = AbstractDataProvider::GenericExpressionImplementations{exp.exp};
        if (!expinfo) {
            throw "INVALID-OPERATION", sprintf("operator %y is unknown; supported "
                "operators: %y", exp.exp, keys GenericExpressionImplementations);
        }
        on_error rethrow $1.err, sprintf("error in return value type %y for %y expression: %s",
            expinfo.exp.return_type.getName(), expinfo.exp.name, $1.desc);
        return expinfo.exp.return_type.acceptsValue(expinfo.impl(rec, exp));
    }

    #! Set callbacks for dynamic URI resolution to allow for variable URI path elements to be resolved at runtime
    /** @param value_needs_resolution signature <tt>bool sub (string value) { ... }</tt>; this callback will be used
        to check if a URI path needs dynamic resolution
        @param resolve_value signature <tt>auto sub (auto value) { ... }</tt>; if \c value_needs_resolution returns
        @ref True on a URI path, then this callback will be called at runtime for each request to resolve the URI path
        to the final path

        @return False if this call was ignored because callbacks are locked, True if the call was successful

        @note after this call, callbacks cannot be changed
    */
    static bool setDynamicValueCallbacks(code value_needs_resolution, code resolve_value) {
        if (!AbstractDataProvider::checkCallbacks()) {
            return value_needs_resolution == cb_value_needs_resolution && resolve_value == cb_resolve_value;
        }
        cb_value_needs_resolution = value_needs_resolution;
        cb_resolve_value = resolve_value;
        return True;
    }

    #! processes an expression in a certain context
    /** @param role the role of the operation being performed
        @param expmap the map of supported expressions for this data provider
        @param caps the logic capabilities supported by the data provider for the current operation
        @param expected_type the type the expression should be compatible with
        @param exp the expression code / label to process

        @throw DATA-PROVIDER-EXPRESSION-ERROR the expression has an error; argument mismatch, type error, etc
    */
    static verifyExpression(int role, *hash<string, hash<DataProviderExpressionInfo>> expmap, int caps,
            AbstractDataProviderType expected_type, hash<DataProviderExpression> exp) {
        hash<DataProviderExpressionInfo> expinfo = AbstractDataProvider::getExpression(role, exp.exp, expmap);
        return AbstractDataProvider::verifyExpression(role, expinfo, caps, expected_type, exp);
    }

    #! processes an expression in a certain context
    /** @param role the role of the operation being performed
        @param expinfo the definition of the expression to evaluate
        @param caps the logic capabilities supported by the data provider for the current operation
        @param expected_type the type the expression should be compatible with
        @param exp the expression code / label to process

        @throw DATA-PROVIDER-EXPRESSION-ERROR the expression has an error; argument mismatch, type error, etc
    */
    static verifyExpression(int role, hash<DataProviderExpressionInfo> expinfo, int caps,
            AbstractDataProviderType expected_type, hash<DataProviderExpression> exp) {
        # check arguments
        if (exp.args.lsize() > expinfo.args.size() && !expinfo.varargs) {
            throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("value provided for expression %y has too many values "
                "(%d); only max %d are accepted", expinfo.label, exp.args.lsize(), expinfo.args.size());
        }
        map AbstractDataProvider::verifyExpressionArgValue(role, caps, expinfo, exp.args, $#), expinfo.args;
        if (exp.args.lsize() > expinfo.args.size()) {
            # process the rest of the arguments
            map AbstractDataProvider::verifyExpressionArgValue(role, caps, expinfo, exp.args, expinfo.args.size() + $#),
                xrange(exp.args.lsize() - expinfo.args.size());
        }
        if (!expected_type.isAssignableFrom(expinfo.return_type)) {
            throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("expecting a %y value, but expression %y returns a %y "
                "value instead", expected_type.getName(), expinfo.label, expinfo.return_type.getName());
        }
    }

    #! processes an expression in a certain context
    /** @param role the role of the operation being performed
        @param caps the logic capabilities supported by the data provider for the current operation
        @param expinfo the expression being processed
        @param values the values / arguments for the expression
        @param pos the position in the argument list
    */
    static verifyExpressionArgValue(int role, int caps, hash<DataProviderExpressionInfo> expinfo,
            softlist<auto> values, int pos) {
        hash<DataProviderSignatureTypeInfo> arginfo = expinfo.args[pos] ?? expinfo.args.last();
        if (arginfo.type_code == ST_Any) {
            return;
        }
        auto val = values[pos];
        bool is_exp = val instanceof hash<DataProviderExpression>;
        bool is_ref = is_exp ? False : val instanceof hash<DataProviderFieldReference>;

        switch (arginfo.type_code) {
            case ST_Any:
                return;
            case ST_Value:
                if (is_exp || is_ref) {
                    throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("expression %y argument %d/%d expects a value; "
                        "got %y instead", expinfo.label, pos + 1, values.size(),
                        is_exp ? "nested expression" : "field reference");
                }
                if (!arginfo.type.isAssignableFrom(Type::getType(val))) {
                    throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("argument %d expects type %y; got %y instead",
                        pos + 1, arginfo.type.getName(), val.fullType());
                }
                return;
            case ST_Field:
                if (!is_ref) {
                    throw "DATA-PROVIDER-EXPRESSION-ERROR", sprintf("expression %y argument %d/%d expects a field "
                        "reference; got %y instead", expinfo.label, pos + 1, values.size(),
                        is_exp ? "nested expression" : "value");
                }
                return;
        }
    }

    #! Checks if callbacks have already been set or locked
    private static bool checkCallbacks() {
        if (callbacks_locked) {
            return False;
        }
        return callbacks_locked = True;
    }

    #! validates create options
    /** @throw CREATE-OPTION-ERROR invalid or unsupported create option
    */
    private *hash<auto> validateCreateOptions(*hash<auto> create_options) {
        return checkOptions("CREATE-OPTION-ERROR", getCreateOptions(), create_options);
    }

    #! validates upsert options
    /** @throw UPSERT-OPTION-ERROR invalid or unsupported upsert option
    */
    private *hash<auto> validateUpsertOptions(*hash<auto> upsert_options) {
        return checkOptions("UPSERT-OPTION-ERROR", getUpsertOptions(), upsert_options);
    }

    #! validates search options
    /** @throw SEARCH-OPTION-ERROR invalid or unsupported search option
    */
    private *hash<auto> validateSearchOptions(*hash<auto> search_options) {
        return checkOptions("SEARCH-OPTION-ERROR", getSearchOptions(), search_options);
    }

    #! validates request options
    /** @throw REQUEST-OPTION-ERROR invalid or unsupported request option
    */
    private *hash<auto> validateRequestOptions(*hash<auto> request_options) {
        return checkOptions("REQUEST-OPTION-ERROR", getRequestOptions(), request_options);
    }

    #! validates child create options
    /** @throw CHILD-CREATE-OPTION-ERROR invalid or unsupported child create option

        @since DataProvider 2.0
    */
    private *hash<auto> validateChildCreateOptions(*hash<auto> child_create_options) {
        return checkOptions("CHILD-CREATE-OPTION-ERROR", getChildCreateOptions(), child_create_options);
    }

    #! validates child delete options
    /** @throw CHILD-DELETE-OPTION-ERROR invalid or unsupported child delete option

        @since DataProvider 2.0
    */
    private *hash<auto> validateChildDeleteOptions(*hash<auto> child_delete_options) {
        return checkOptions("CHILD-DELETE-OPTION-ERROR", getChildDeleteOptions(), child_delete_options);
    }

    #! validates field add options
    /** @throw FIELD-ADD-OPTION-ERROR invalid or unsupported field add option

        @since DataProvider 2.0
    */
    private *hash<auto> validateFieldAddOptions(*hash<auto> field_add_options) {
        return checkOptions("FIELD-ADD-OPTION-ERROR", getFieldAddOptions(), field_add_options);
    }

    #! validates field update options
    /** @throw FIELD-UPDATE-OPTION-ERROR invalid or unsupported field update option

        @since DataProvider 2.0
    */
    private *hash<auto> validateFieldUpdateOptions(*hash<auto> field_update_options) {
        return checkOptions("FIELD-UPDATE-OPTION-ERROR", getFieldUpdateOptions(), field_update_options);
    }

    #! validates field delete options
    /** @throw FIELD-DELETE-OPTION-ERROR invalid or unsupported field delete option

        @since DataProvider 2.0
    */
    private *hash<auto> validateFieldDeleteOptions(*hash<auto> field_delete_options) {
        return checkOptions("FIELD-DELETE-OPTION-ERROR", getFieldDeleteOptions(), field_delete_options);
    }

    #! validates send message options
    /** @throw SEND-MESSAGE-OPTION-ERROR invalid or unsupported send message option

        @since DataProvider 2.5
    */
    private *hash<auto> validateSendMessageOptions(*hash<auto> send_message_options) {
        return checkOptions("SEND-MESSAGE-OPTION-ERROR", getSendMessageOptions(), send_message_options);
    }

    #! processes options passed to the constructor
    private processConstructorOptions(*hash<string, hash<DataProviderOptionInfo>> option_desc, *hash<auto> options) {
        self += checkOptions("CONSTRUCTOR-OPTION-ERROR", option_desc, options);
    }

    #! verifies options according to the option
    private *hash<auto> checkOptions(string err, *hash<string, hash<DataProviderOptionInfo>> option_desc,
            *hash<auto> uoptions) {
        # to ensure that the return value has type "hash<auto>"
        hash<auto> options += uoptions;
        if (!option_desc && options) {
            error(err, "invalid options: %y; no options are supported",
                keys options);
        }
        if (*list<string> invalid_keys = keys (options - (keys option_desc))) {
            error(err, "invalid options: %y; supported options: %y", invalid_keys, keys option_desc);
        }
        # check types and required keys
        foreach hash<auto> i in (option_desc.pairIterator()) {
            bool present = options.hasKey(i.key);
            if (!present) {
                if (exists i.value.default_value) {
                    options{i.key} = i.value.default_value;
                    continue;
                } else if (i.value.required) {
                    error(err, "missing required option %y%s", i.key,
                        i.value.desc ? sprintf(" (%s)", i.value.desc) : "");
                }
                continue;
            }
            # process value
            bool ok;
            string err;
            foreach AbstractDataProviderType type in (i.value.type) {
                try {
                    options{i.key} = type.acceptsValue(options{i.key});
                    ok = True;
                    break;
                } catch (hash<ExceptionInfo> ex) {
                    if (ex.err == "RUNTIME-TYPE-ERROR") {
                        err = sprintf("option %y of type %y cannot be assigned a value of type %y: %s: %s", i.key,
                            type.getName(), options{i.key}.type(), ex.err, ex.desc);
                        continue;
                    }
                    rethrow;
                }
            }
            if (!ok) {
                if (err) {
                    error("RUNTIME-TYPE-ERROR", err);
                }
                error("RUNTIME-TYPE-ERROR", "no type information available for option %y", i.key);
            }
            # check allowed values
            if (i.value.allowed_values) {
                bool ok = False;
                foreach auto v in (i.value.allowed_values) {
                    if (options{i.key} === v.value) {
                        ok = True;
                        break;
                    }
                }
                if (!ok) {
                    error("OPTION-VALUE-ERROR", sprintf("option %y does not accept value %y; possible values: %y",
                        i.key, options{i.key}, (map $1.value, i.value.allowed_values)));
                }
            }
        }
        return options;
    }

    #! processes search parameter expressions
    /** @param exp the expression to process
        @param search_options the search options after processing by validateSearchOptions()

        @return the processed expression
    */
    private *hash<DataProviderExpression> processSearchParameters(*hash<DataProviderExpression> exp,
            *hash<auto> search_options) {
        if (!exp) {
            return exp;
        }
        checkSearchExpressions();
        hash<DataProviderInfo> info = getInfo();
        verifyExpression(ER_Search, info.expressions, info.search_logic_capabilities,
            AbstractDataProviderTypeMap."bool", exp);
        return exp;
    }

    #! processes search or set values to convert types if necessary
    /** @param h the hash of field values to process
        @param search_options the search options after processing by validateSearchOptions()

        @return processed field values, all non-hash values of \a h are processed with the field's type to ensure
        that the values are valid for the comparison or set operation; hash values are assumed to be provider-
        specific comparison or set operations and should be processed by the provider-specific method
    */
    private *hash<auto> processFieldValues(*hash<auto> h, *hash<auto> search_options) {
        if (!h) {
            return h;
        }
        *hash<string, AbstractDataField> rec = getRecordTypeImpl(search_options);
        hash<auto> rv;
        foreach hash<auto> i in (h.pairIterator()) {
            if (!rec{i.key}) {
                list<string> f = map regex_subst($1, "\\\\.", ".", RE_Global), i.key.splitRegex("(?<!\\\\)\\.");
                if (rec && !rec{f[0]}) {
                    error("INVALID-FIELD", "field %y is not a defined field; valid fields: %y", f[0], keys rec);
                }
                rv{i.key} = i.value;
                continue;
            }
            # we assume a hash is a provider-specific search option
            if (i.value.typeCode() == NT_HASH) {
                rv{i.key} = i.value;
                continue;
            }
            rv{i.key} = rec
                ? rec{i.key}.acceptsValue(i.value)
                : i.value;
        }
        return rv;
    }

    #! thrown an exception
    private error(string err, string fmt) {
        string desc_name;
        # the getName() method cannot be run in the constructor before the object has been created and the name set
        try {
            desc_name = sprintf("%y (%y)", getName(), self.className());
        } catch (hash<ExceptionInfo> ex) {
            # this block intentionally left blank
        }
        if (!desc_name) {
            desc_name = sprintf("class %y", self.className());
        }
        throw err, sprintf("data provider %s: %s", desc_name, vsprintf(fmt, argv));
    }

    #! Returns the first record matching the search options
    /** @param where_cond the search criteria
        @param search_options the search options after processing by validateSearchOptions()

        @note This default method returns immediately as soon as the first matching record is found
    */
    private *hash<auto> searchFirstRecordImpl(hash<auto> where_cond,
            *hash<auto> search_options) {
        # scans the whole data set to find a single record
        AbstractDataProviderRecordIterator i = searchRecordsImpl(where_cond, search_options);
        if (i.next()) {
            return i.getValue();
        }
    }

    #! Returns a single record matching the search options
    /** @param where_cond the search criteria
        @param search_options the search options after processing by validateSearchOptions()

        @throw MULTIPLE-RECORDS-ERROR multiple records found

        @note This default methods searches the entire data set for a matching record
    */
    private *hash<auto> searchSingleRecordImpl(hash<auto> where_cond,
            *hash<auto> search_options) {
        # scans the whole data set to find a single record
        *list<*hash<auto>> records = map $1, searchRecordsImpl(where_cond, search_options);
        if (records.lsize() > 1) {
            error("MULTIPLE-RECORDS-ERROR", "%d records found for search: %y", records.size(), where_cond);
        }
        return records[0];
    }

    #! Returns a list of child data provider names, if any
    /** @return a list of child data provider names, if any

        This base class method returns @ref nothing
    */
    private *list<string> getChildProviderNamesImpl() {
        # this method intentionally left empty
    }

    #! Returns the given child provider or @ref nothing if the given child is unknown
    /** @return the given child provider or @ref nothing if the given child is unknown
    */
    private *AbstractDataProvider getChildProviderImpl(string name) {
        # this method intentionally left empty
    }

    #! Returns the description of the record type, if any
    /** @param search_options to be included and processed by validateSearchOptions() if
        \a recordRequiresSearchOptions() is @ref True for this provider, otherwise any value provided in this argument
        is ignored

        @return the record type or @ref NOTHING if the datas provider does not support a record type
    */
    *hash<string, AbstractDataField> getRecordType(*hash<auto> search_options) {
        if (!hasRecord()) {
            return;
        }
        return getRecordTypeImpl(recordRequiresSearchOptions() ? validateSearchOptions(search_options) : NOTHING);
    }

    #! Returns the description of the record type with soft types, if any
    *hash<string, AbstractDataField> getSoftRecordType(*hash<auto> search_options) {
        return map {
            $1.key: $1.value.getSoftType()
        }, getRecordType(search_options).pairIterator();
    }

    #! Returns the description of the record type with "or nothing" types, if any
    *hash<string, AbstractDataField> getOrNothingRecordType(*hash<auto> search_options) {
        return map {
            $1.key: $1.value.getOrNothingType(True)
        }, getRecordType(search_options).pairIterator();
    }

    #! Returns options that can be used for searching
    /** @return a hash of options that can be used for searching; keys are search option names, values describe the
        search option; if @ref nothing is returned, then the provider does not support any search options
    */
    *hash<string, hash<DataProviderOptionInfo>> getSearchOptions() {
        return getStaticInfoImpl().search_options;
    }

    #! Returns options that can be used for creating records
    /** @return a hash of options that can be used for creating records; keys are option names, values describe the
        option; if @ref nothing is returned, then the provider does not support any creation options
    */
    *hash<string, hash<DataProviderOptionInfo>> getCreateOptions() {
        return getStaticInfoImpl().create_options;
    }

    #! Returns options that can be used for upserting records
    /** @return a hash of options that can be used for upserting records; keys are option names, values describe the
        option; if @ref nothing is returned, then the provider does not support any upsert options
    */
    *hash<string, hash<DataProviderOptionInfo>> getUpsertOptions() {
        return getStaticInfoImpl().upsert_options;
    }

    #! Returns options that can be used for requests
    /** @return a hash of options that can be used for requests; keys are request option names, values describe the
        request option; if @ref nothing is returned, then the provider does not support any request options
    */
    *hash<string, hash<DataProviderOptionInfo>> getRequestOptions() {
        return getStaticInfoImpl().request_options;
    }

    #! Returns options that can be used for creating child data providers
    /** @return a hash of options that can be used for creating child data providers; keys are option names, values
        describe the option; if @ref nothing is returned, then the provider does not support any child creation
        options

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> getChildCreateOptions() {
        return getStaticInfoImpl().child_create_options;
    }

    #! Returns options that can be used for deleting child data providers
    /** @return a hash of options that can be used for delete child data providers; keys are option names, values
        describe the option; if @ref nothing is returned, then the provider does not support any child deletion
        options

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> getChildDeleteOptions() {
        return getStaticInfoImpl().child_delete_options;
    }

    #! Returns options that can be used for adding new fields to the data provider
    /** @return a hash of options that can be used for adding new fields; keys are option names, values
        describe the option; if @ref nothing is returned, then the provider does not support any add field
        options

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> getFieldAddOptions() {
        return getStaticInfoImpl().add_field_options;
    }

    #! Returns options that can be used for updating fields of the data provider
    /** @return a hash of options that can be used for updating fields; keys are option names, values
        describe the option; if @ref nothing is returned, then the provider does not support any update field
        options

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> getFieldUpdateOptions() {
        return getStaticInfoImpl().update_field_options;
    }

    #! Returns options that can be used for deleting fields from the data provider
    /** @return a hash of options that can be used for deleting fields; keys are option names, values
        describe the option; if @ref nothing is returned, then the provider does not support any delete field
        options

        @since DataProvider 2.0
    */
    *hash<string, hash<DataProviderOptionInfo>> getFieldDeleteOptions() {
        return getStaticInfoImpl().delete_field_options;
    }

    #! Returns options that can be used for sending messages from the data provider
    /** @return a hash of options that can be used for sending messages; keys are option names, values
        describe the option; if @ref nothing is returned, then the provider does not support any message send
        options

        @since DataProvider 2.5
    */
    *hash<string, hash<DataProviderOptionInfo>> getSendMessageOptions() {
        return getStaticInfoImpl().send_message_options;
    }

    #! Returns a code describing how the data provider supports sending messages
    /** @since DataProvider 2.5
    */
    string getMessageSupport() {
        return getStaticInfoImpl().supports_messages;
    }

    #! Returns @ref True if the data provider supports reading
    /**
    */
    bool supportsRead() {
        return getStaticInfoImpl().supports_read;
    }

    #! Returns @ref True if the data provider supports native bulk reading
    /**
    */
    bool supportsBulkRead() {
        return getStaticInfoImpl().supports_bulk_read;
    }

    #! Returns @ref True if the data provider supports the record creation API
    /**
    */
    bool supportsCreate() {
        return getStaticInfoImpl().supports_create;
    }

    #! Returns @ref True if the data provider supports the record update API
    /**
    */
    bool supportsUpdate() {
        return getStaticInfoImpl().supports_update;
    }

    #! Returns @ref True if the data provider supports the record upsert API
    /**
    */
    bool supportsUpsert() {
        return getStaticInfoImpl().supports_upsert;
    }

    #! Returns @ref True if the data provider supports the record deletion API
    /**
    */
    bool supportsDelete() {
        return getStaticInfoImpl().supports_delete;
    }

    #! Returns @ref True if the data provider supports the record search API natively
    /**
    */
    bool supportsNativeSearch() {
        return getStaticInfoImpl().supports_native_search;
    }

    #! Returns True if the data provider supports bulk creation output
    /**
    */
    bool supportsBulkCreate() {
        return getStaticInfoImpl().supports_bulk_create;
    }

    #! Returns True if the data provider supports bulk upserts
    /**
    */
    bool supportsBulkUpsert() {
        return getStaticInfoImpl().supports_bulk_upsert;
    }

    #! Returns True if the data provider supports requests
    /**
    */
    bool supportsRequest() {
        return getStaticInfoImpl().supports_request;
    }

    #! Returns True if the data provider requires search options to retrieve the record type
    /**
    */
    bool recordRequiresSearchOptions() {
        return getStaticInfoImpl().record_requires_search_options;
    }

    #! Returns @ref True if the data provider has a record type
    /** @return @ref True if the data provider has a record type
    */
    bool hasRecord() {
        return getStaticInfoImpl().has_record;
    }

    #! Returns True if the data provider supports creating new child data providers
    /** @since DataProvider 2.0
    */
    bool supportsCreateChild() {
        return getStaticInfoImpl().supports_child_create;
    }

    #! Returns True if the data provider supports creating deleting child data providers
    /** @since DataProvider 2.0
    */
    bool supportsDeleteChild() {
        return getStaticInfoImpl().supports_child_delete;
    }

    #! Returns True if the data provider supports creating adding new fields
    /** @since DataProvider 2.0
    */
    bool supportsAddField() {
        return getStaticInfoImpl().supports_add_field;
    }

    #! Returns True if the data provider supports updating existing fields
    /** @since DataProvider 2.0
    */
    bool supportsUpdateField() {
        return getStaticInfoImpl().supports_update_field;
    }

    #! Returns True if the data provider supports deleting existing fields
    /** @since DataProvider 2.0
    */
    bool supportsDeleteField() {
        return getStaticInfoImpl().supports_delete_field;
    }

    #! Returns True if the data provider supports a schema
    /** @since DataProvider 2.2
    */
    bool supportsSchema() {
        return getStaticInfoImpl().supports_schema;
    }

    #! Returns True if the data provider supports advanced search expressions
    /** @since DataProvider 2.2
    */
    bool supportsSearchExpressions() {
        return getStaticInfoImpl().supports_search_expressions;
    }

    #! Returns True if the data provider supports the observer pattern / event API
    /** @since DataProvider 2.5
    */
    bool supportsObservable() {
        return getStaticInfoImpl().supports_observable;
    }

    #! Returns True if the data provider supports sending messages
    /** @since DataProvider 2.5
    */
    bool supportsMessages() {
        return getStaticInfoImpl().supports_messages != MSG_None;
    }

    #! Returns the schema type for this data provider or @ref nothing if no schema type is supported
    /** @since DataProvider 2.2
    */
    *object getSchemaType() {
        return getStaticInfoImpl().schema_type;
    }

    #! Returns the description of the record type, if any
    /** @param search_options to be included and processed by validateSearchOptions() if
        \a recordRequiresSearchOptions() is @ref True for this provider, otherwise any value provided in this argument
        is ignored
    */
    private *hash<string, AbstractDataField> getRecordTypeImpl(*hash<auto> search_options) {
        throwUnimplementedException();
    }

    #! Returns an iterator for zero or more records matching the search options
    /** @param block_size the number of records in a read block; must be a positive number
        @param where_cond the search criteria; after processing by processSearchParameters()
        @param search_options the search options after processing by validateSearchOptions()

        @throw INVALID-BLOCK-SIZE the block size must be a positive number
        @throw INVALID-OPERATION the data provider does not support reading
    */
    private AbstractDataProviderBulkRecordInterface searchRecordsBulkImpl(int block_size = 1000,
            *hash<auto> where_cond, *hash<auto> search_options) {
        return new DefaultBulkRecordIterface(block_size, searchRecordsImpl(where_cond, search_options));
    }

    #! Returns an iterator for zero or more records matching the search options
    /** @param where_cond the search criteria; after processing by processSearchParameters()
        @param search_options the search options after processing by validateSearchOptions()

        @see requestSearchRecordsImpl()
    */
    private AbstractDataProviderRecordIterator searchRecordsImpl(*hash<auto> where_cond,
            *hash<auto> search_options) {
        throwUnimplementedException();
    }

    #! Returns an iterator for zero or more records matching the search options according to an API request
    /** @param req the request to serialize and make according to the request type
        @param where_cond the search criteria; after processing by processSearchParameters()
        @param search_options the search options after processing by validateSearchOptions()

        This will execute the request and perform a default search on any record(s) returned

        @see searchRecordsImpl()
    */
    private AbstractDataProviderRecordIterator requestSearchRecordsImpl(auto req,
            *hash<auto> where_cond, *hash<auto> search_options) {
        throwUnimplementedException();
    }

    #! Creates the given record to the data provider
    /** @param rec a hash representing a single input record
        @param create_options the create options after processing by validateCreateOptions()

        @return the data written to the data provider

        @throw DUPLICATE-RECORD this exception should be thrown if the provider fails due to an attempt to create a
        duplicate record
    */
    private *hash<auto> createRecordImpl(hash<auto> rec, *hash<auto> create_options) {
        throwUnimplementedException();
    }

    #! Upserts the given record to the data provider
    /** @param rec a hash representing a single input record
        @param upsert_options the create options after processing by validateUpsertOptions()

        @return see @ref db_provider_upsert_results for possible values
    */
    private string upsertRecordImpl(hash<auto> rec, *hash<auto> upsert_options) {
        throwUnimplementedException();
    }

    #! Updates a single record matching the search options
    /** @param set the hash of field data to set
        @param where_cond the search criteria
        @param search_options the search options after processing by validateSearchOptions()
    */
    private bool updateSingleRecordImpl(hash<auto> set, hash<auto> where_cond,
            *hash<auto> search_options) {
        throwUnimplementedException();
    }

    #! Updates zero or more records matching the search options
    /** @param set the hash of field data to set
        @param where_cond a hash for identifying the record(s) to be updated
        @param search_options the update options after processing by validateSearchOptions()

        @return the number of records updated
    */
    private int updateRecordsImpl(hash<auto> set, *hash<auto> where_cond,
            *hash<auto> search_options) {
        throwUnimplementedException();
    }

    #! Deletes zero or more records
    /**
        @param where_cond a hash for identifying the record(s) to be deleted
        @param search_options the delete options after processing by validateSearchOptions()

        @return the number of records deleted
    */
    private int deleteRecordsImpl(*hash<auto> where_cond, *hash<auto> search_options) {
        throwUnimplementedException();
    }

    #! Makes a request and returns the response
    /** @param req the request to serialize and make according to the request type
        @param request_options the request options; will be processed by validateRequestOptions()

        @return the response to the request
    */
    private auto doRequestImpl(auto req, *hash<auto> request_options) {
        throwUnimplementedException();
    }

    #! Creates a new child data provider and returns it after adding as a child
    /** @param name the name of the new child data provider
        @param fields the fields for the new child data provider
        @param child_create_options the options for creating the new child data provider

        @return the new child data provider

        @since DataProvider 2.0
    */
    private AbstractDataProvider createChildProviderImpl(string name, hash<string, AbstractDataField> fields,
            *hash<auto> child_create_options) {
        throwUnimplementedException();
    }

    #! Deletes a child data provider
    /** @param name the name of the new child data provider
        @param child_delete_options the options for deleting a child data provider

        @since DataProvider 2.0
    */
    private deleteChildProviderImpl(string name, *hash<auto> child_delete_options) {
        throwUnimplementedException();
    }

    #! Creates a new field
    /**
        @param field the field to add
        @param field_add_options the options for adding the new field

        @since DataProvider 2.0
    */
    private addFieldImpl(AbstractDataField field, *hash<auto> field_add_options) {
        throwUnimplementedException();
    }

    #! Updates an existing field
    /** @param name the name of the current field to update
        @param field the new definition of the field
        @param field_update_options the options for adding the new field

        @since DataProvider 2.0
    */
    private updateFieldImpl(string name, AbstractDataField field, *hash<auto> field_update_options) {
        throwUnimplementedException();
    }

    #! Deletes an existing field
    /** @param name the name of the field to delete
        @param field_delete_options the options for deleting the new field

        @since DataProvider 2.0
    */
    private deleteFieldImpl(string name, *hash<auto> field_delete_options) {
        throwUnimplementedException();
    }

    #! Sends a message from message-capable data providers
    /**
        @param message_id the message ID to send; must correspond to a valid message ID
        @param msg the message to send
        @param send_message_options the options for sending the message

        @since DataProvider 2.5
    */
    private sendMessageImpl(string message_id, auto msg, *hash<auto> send_message_options) {
        throwUnimplementedException();
    }

    #! Returns the schema object supporting this data provider
    /** @return the schema object supporting this data provider

        @since DataProvider 2.2
    */
    private object getSchemaObjectImpl() {
        throwUnimplementedException();
    }

    #! Returns the description of a successful request message, if any
    /** @return the request type for this provider

        @note only called if the provider supports the request / response API
    */
    private *AbstractDataProviderType getRequestTypeImpl() {
        throwUnimplementedException();
    }

    #! Returns the description of a response message, if this object represents a response message
    /** @return the response type for this response message

        @note only called if the provider supports the request / response API
    */
    private *AbstractDataProviderType getResponseTypeImpl() {
        throwUnimplementedException();
    }

    #! Returns a hash of error responses, if any
    /** @return a hash of error responses, if any, keyed by response code or string

        @note only called if the provider supports the request / response API
    */
    private *hash<string, AbstractDataProviderType> getErrorResponseTypesImpl() {
        throwUnimplementedException();
    }

    #! Returns the type for the given error code
    /** @param error_code the error code for the response; must be a known error code, or an \c UNKNOWN-ERROR-RESPONSE
        exception is thrown
        @return the type for the given error code

        @throw UNKNOWN-ERROR-RESPONSE the error response given is not known
    */
    private AbstractDataProviderType getErrorResponseTypeImpl(string error_code) {
        # note: this should be reimplemented with a more efficient implementation if creating the error type hash on
        # demand is expensive
        *hash<string, AbstractDataProviderType> err_map = getErrorResponseTypesImpl();
        *AbstractDataProviderType type = err_map{error_code};
        if (!type) {
            throw "UNKNOWN-ERROR-RESPONSE", sprintf("error code %y is unknown; known error codes: %y", error_code,
                keys err_map);
        }
        return type;
    }

    #! Returns the type of an observable event, if any
    /** @param event_id the event ID of the event

        @return the observable event type for this provider

        @note only called if the provider supports the observer pattern / event API

        @throw INVALID-EVENT the event is not supported by this data provider

        @since DataProvider 2.5
    */
    private hash<DataProviderMessageInfo> getEventInfoImpl(string event_id) {
        *hash<string, hash<DataProviderMessageInfo>> h = getEventTypes();
        if (!h{event_id}) {
            throw "INVALID-EVENT", sprintf("provider %y (type %y) does not support event %y; supported events: %y",
                getName(), self.className(), event_id, keys h);
        };
        return h{event_id};
    }

    #! Returns a hash of all supported event types
    /** @return a hash of all supported event types

        @note only called if the provider supports the observer pattern / event API

        @since DataProvider 2.5
    */
    private hash<string, hash<DataProviderMessageInfo>> getEventTypesImpl() {
        throwUnimplementedException();
    }

    #! Returns the type of an outbound message, if any
    /** @param message_id the ID of the message

        @return the message type for this provider

        @throw INVALID-MESSAGE the message is not supported by this data provider

        @since DataProvider 2.5
    */
    private hash<DataProviderMessageInfo> getMessageInfoImpl(string message_id) {
        *hash<string, hash<DataProviderMessageInfo>> h = getMessageTypes();
        if (!h{message_id}) {
            throw "INVALID-MESSAGE", sprintf("provider %y (type %y) does not support message %y; supported "
                "messages: %y", getName(), self.className(), message_id, keys h);
        };
        return h{message_id};
    }

    #! Returns a hash of all supported outbound messages
    /** @return a hash of all supported outbound messages

        @throw INVALID-OPERATION the data provider does not support sending messages

        @since DataProvider 2.5
    */
    private hash<string, hash<DataProviderMessageInfo>> getMessageTypesImpl() {
        throwUnimplementedException();
    }

    #! Throws an \c INVALID-OPERATION exception
    /** @throw INVALID-OPERATION this exception is thrown unconditionally by this method
    */
    private throwUnimplementedException() {
        # should never be called
        throw "UNIMPLEMENTED", sprintf("data provider %y of class %y has an error; the operation is not supported; "
            "this indicates a bug in the implementation of the data provider; please report this bug to the author "
            "of the data provider",
            getName(), self.className());
    }

    #! Returns the data provider name
    abstract string getName();

    #! Returns data provider static info
    private abstract hash<DataProviderInfo> getStaticInfoImpl();
}
}
