#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

%require-types
%enable-all-warnings
%new-style
%strict-args

%requires ../../../../qlib/Util.qm
%requires ../../../../qlib/QUnit.qm
%requires ../../../../qlib/MapperUtil.qm
%requires ../../../../qlib/Logger.qm
%requires ../../../../qlib/DataProvider
%requires ../../../../qlib/Mapper.qm

%exec-class MapperTest

class TestOutputHashType inherits HashDataType {
    constructor() {
        addField(new QoreDataField("stringval", NOTHING, SoftStringType));
    }
}

class TestHashType inherits HashDataType {
    constructor() {
        addField(new QoreDataField("output", NOTHING, new TestOutputHashType()));
    }
}

class TestInputProvider inherits AbstractDataProvider {
    private {
        *hash<auto> last_where_cond;
        *hash<auto> last_search_options;
    }

    string getName() {
        return "TestInputProvider";
    }

    hash<DataProviderInfo> getStaticInfoImpl() {
        return <DataProviderInfo>{
            "supports_read": True,
            "has_record": True,
            "expressions": AbstractDataProvider::GenericExpressions,
        };
    }

    *hash<auto> getLastWhereCond() {
        return last_where_cond;
    }

    *hash<auto> getLastSearchOptions() {
        return last_search_options;
    }

    private *hash<string, AbstractDataField> getRecordTypeImpl(*hash<auto> search_options) {
        return {
            "sub-a": new QoreDataField("sub-a", "a", AbstractDataProviderType::get("string")),
            "sub-b": new QoreDataField("sub-b", "b", AbstractDataProviderType::get("string")),
        };
    }

    AbstractDataProviderRecordIterator searchRecords(*hash<auto> where_cond, *hash<auto> search_options) {
        last_where_cond = where_cond;
        return AbstractDataProvider::searchRecords(where_cond, search_options);
    }

    private AbstractDataProviderRecordIterator searchRecordsImpl(*hash<auto> where_cond, *hash<auto> search_options) {
        last_search_options = search_options;
        return new DefaultRecordIterator(new HashListIterator({"sub-a": ("a-0", "a-1",), "sub-b": ("b-0", "b-1",)}),
            where_cond, search_options, getRecordTypeImpl());
    }
}

class TestOutputProvider inherits AbstractDataProvider {
    string getName() {
        return "TestOutputProvider";
    }

    hash<DataProviderInfo> getStaticInfoImpl() {
        return <DataProviderInfo>{
            "supports_create": True,
            "supports_upsert": True,
            "has_record": True,
        };
    }

    private *hash<string, AbstractDataField> getRecordTypeImpl(*hash<auto> search_options) {
        return {
            "sub-a": new QoreDataField("sub-a", "a", AbstractDataProviderType::get("string")),
            "sub-b": new QoreDataField("sub-b", "b", AbstractDataProviderType::get("string")),
        };
    }
}

public class MapperTest inherits QUnit::Test {
    public {
        const DataMap = {
            # this will take the "Id" element of any "^attributes^" hash in the input record
            "id": "^attributes^.Id",
            # this maps input "name" -> output "name"
            "name": True,
            # this marks "explicit_count" as an integer field mapped from the input "Count" field
            "explicit_count": ("type": "int", "name": "Count"),
            # runs the given code on the input record and retuns the result - the number of "Products" sub-records
            "implicit_count": int sub (auto ignored, hash<auto> rec) { return rec.Products.size(); },
            # converts the given field to a date in the specified format
            "order_date": ("name": "OrderDate", "date_format": "DD.MM.YYYY HH:mm:SS.us"),
            # converts the given field to a number in the specified format
            "unit_price": {"name": "UnitPrice", "type_options": {"number.format": ".,"}},
            # returns a constant value
            "target_sys": ("constant": "Orders"),
            # returns structured output
            "sr0.sr1.key0": ("constant": "key0"),
            "sr0.sr1.^attributes^.key0": ("constant": "key0"),
            "sr0.sr1.^attributes^.key1": ("constant": "key1"),
            "sr0.sr1.^attributes^.type": (
                "name": "Type",
                "code": *string sub (*string v, hash<auto> rec) { return v ? v.lwr() : NOTHING;},
                "default": "unknown",
            ),
            "sr0.sr1.key1": ("constant": "key1"),
            "sr0.store_name": "StoreInfo.StoreName",
            "runtimetest": ("runtime" : "runtimetest"),
            "indextest1": ("index": 0),
            "indextest2": ("index": 1),
            "indextest3": ("index": "a"),
            "empty_string": "EmptyString",
            "length_test": ("maxlen": 5, "default": ""),
        };

        const DataMap2 = {
            "int_field": ("type": "int", "name": "count"),
            "date_field": ("name": "idate"),
        };

        const MapInput = ((
            "^attributes^": ("Id": 1),
            "name": "John Smith",
            "Count": 1,
            "OrderDate": "02.01.2014 10:37:45.103948",
            "UnitPrice": "1.543,50",
            "StoreInfo": ("StoreName": "Store1"),
            "EmptyString" : "",
            "Products": (
                {
                    "ProductName": "Widget 1",
                    "Quantity": 1,
                },
            ),
            "length_test": NOTHING,
            ), (
            "^attributes^": ("Id": 2),
            "name": "Steve Austin",
            "Type": "Retail",
            "Count": 2,
            "OrderDate": "04.01.2014 19:21:08.882634",
            "UnitPrice": "9,95",
            "StoreInfo": ("StoreName": "Store2"),
            "EmptyString" : "",
            "Products": ((
                "ProductName": "Widget X",
                "Quantity": 4,
                ), (
                "ProductName": "Widget 2",
                "Quantity": 2,
                ),
            )),
        );

        # structured as hash of lists
        const MapInput2 = (
            "^attributes^": (("Id": 1), ("Id" : 2)),
            "name": ("John Smith", "Steve Austin"),
            "Type" : (NOTHING, "Retail"),
            "Count": (1, 2),
            "OrderDate": ("02.01.2014 10:37:45.103948",
                          "04.01.2014 19:21:08.882634",),
            "UnitPrice": ("1.543,50", "9,95",),
            "StoreInfo": (("StoreName": "Store1"),
                          ("StoreName": "Store2")),
            "EmptyString" : ("",""),
            "Products": (((
                "ProductName": "Widget 1",
                "Quantity": 1,
                ),),
                ((
                "ProductName": "Widget X",
                "Quantity": 4,
                ), (
                "ProductName": "Widget 2",
                "Quantity": 2,
                ),)),
        );

        const MapInput3 = (
            "count": (11, "10", NOTHING),
            "idate": ("10.12.2019", "11.12.2019", "12.12.2019"),
        );

        const MapOutput = ((
            "id": 1,
            "name": "John Smith",
            "explicit_count": 1,
            "implicit_count": 1,
            "order_date": 2014-01-02T10:37:45.103948,
            "unit_price": 1543.50n,
            "target_sys": "Orders",
            "sr0": (
                "sr1": (
                    "key0": "key0",
                    "^attributes^": (
                        "key0": "key0",
                        "key1": "key1",
                        "type": "unknown",
                    ),
                    "key1": "key1",
                ),
                "store_name": "Store1",
            ),
            "runtimetest" : 1,
            "indextest1" : 0,
            "indextest2" : 1,
            "indextest3" : "a0",
            "empty_string" : "",
            "length_test": "",
            ), (
            "id": 2,
            "name": "Steve Austin",
            "explicit_count": 2,
            "implicit_count": 2,
            "order_date": 2014-01-04T19:21:08.882634,
            "unit_price": 9.95n,
            "target_sys": "Orders",
            "sr0": (
                "sr1": (
                    "key0": "key0",
                    "^attributes^": (
                        "key0": "key0",
                        "key1": "key1",
                        "type": "retail",
                    ),
                    "key1": "key1",
                ),
                "store_name": "Store2",
            ),
            "runtimetest" : 1,
            "indextest1" : 1,
            "indextest2" : 2,
            "indextest3" : "a1",
            "empty_string" : "",
            "length_test": "",
            ),
        );

        const MapOutput2 = (
            ("int_field": 11, "date_field": 2019-12-10),
            ("int_field": 10, "date_field": 2019-12-11),
            ("int_field": NOTHING, "date_field": 2019-12-12),
        );

        const DotMap = (
            "output.1": "input.2",
            "output.2": "input.1",
            "output.3": ("struct": ("a", "b")),
        );

        const DotInput = (
            ("input.1": "abc", "input.2": 123, "a": ("b": 1)),
            ("input.1": "xyz", "input.2": 456, "a": ("b": 2)),
        );

        const DotOutput = (
            ("output.1": 123, "output.2": "abc", "output.3": 1),
            ("output.1": 456, "output.2": "xyz", "output.3": 2),
        );

        const DotMap2 = {
            "output\\.1": "input\\.2",
            "output\\.2": "input\\.1",
            "output\\.3": {"struct": ("a", "b")},
        };

        const OStructMap = (
            "a.id": "input.2",
            "a.other": "input.1",
            "a.something": ("struct": ("a", "b")),
        );

        const OStructInput = (
            ("input.1": "abc", "input.2": 123, "a": ("b": 1)),
            ("input.1": "xyz", "input.2": 456, "a": ("b": 2)),
        );

        const OStructOutput = (
            ("a": ("id": 123, "other": "abc", "something": 1)),
            ("a": ("id": 456, "other": "xyz", "something": 2)),
        );

        const TypeMap0 = {
            "output_0": {
                "name": "input_0",
            },
            "output_1": {
                "name": "input_1",
            },
        };

        const TypeOpts0 = {
            "input": {
                "input_0": {
                    "desc": "input field 0",
                    "type": "hash",
                },
                "input_1": {
                    "desc": "input field 1",
                    "type": "any",
                },
            },
            "output": {
                "output_0": {
                    "desc": "input field 0",
                    "type": "hash",
                },
                "output_1": {
                    "desc": "output field 1",
                    "type": "any",
                },
                "output_2": {
                    "desc": "field desc",
                    "type": "*softstring",
                },
            },
        };

        const TypeMap1 = {
            "output_0": {
                "name": "input_0",
                "type": "hash",
            },
            "output_1": {
                "name": "input_1",
                "type": "any",
            },
        };

        const TypeOpts1 = {
            "input": {
                "input_0": {
                    "desc": "input field 0",
                    "type": "hash",
                },
                "input_1": {
                    "desc": "input field 1",
                    "type": "any",
                },
            },
            "output": {
                "output_0": {
                    "desc": "input field 0",
                },
                "output_1": {
                    "desc": "output field 1",
                },
            },
        };

        const TypeMap2 = {
            "output_0.key1": {
                "name": "input_0.k1",
            },
            "output_0.key2": {
                "name": "input_0.k2",
            },
            "output_1": {
                "name": "input_1",
            },
        };

        const TypeOpts2 = {
            "input": {
                "input_0": {
                    "desc": "input field 0",
                    "type": "hash",
                },
                "input_1": {
                    "desc": "input field 1",
                    "type": "any",
                },
            },
            "output": {
                "output_0": {
                    "type": "hash",
                    "desc": "input field 0",
                },
                "output_1": {
                    "type": "any",
                    "desc": "output field 1",
                },
            },
        };

        const TypeMap3 = {
            "out0": {
                "name": "inp0",
            },
            "out1": {
                "context0": "contextstr0",
            },
            "out2": {
                "context1": "contextstr1",
                "name": "inp1",
            },
        };

        const TypeOpts3 = {
            "input": {
                "inp0": {
                    "type": "string",
                    "desc": "input",
                },
                "inp1": {
                    "type": "string",
                    "desc": "input",
                }
            },
            "output": {
                "out0": {
                    "type": "string",
                },
                "out1": {
                    "type": "string",
                },
                "out2": {
                    "type": "string",
                },
            },
            "runtime_keys": {
                "context0": <MapperRuntimeKeyInfo>{
                    "handler": auto sub (auto arg, hash<auto> ctx, *reference<bool> missing) {
                        return arg + "-ctx";
                    },
                },
                "context1": <MapperRuntimeKeyInfo>{
                    "requires_roles": "value",
                    "handler": auto sub (auto arg, auto val, hash<auto> ctx, *reference<bool> missing) {
                        return val + "-" + arg + "-ctx";
                    },
                },
            },
        };

        const TypeMap3_1 = TypeMap3 + {
            "out1": {
                "context0": "contextstr0",
                "context1": 1,
            },
            "out2": {
                "context1": 2,
                "name": "inp1",
            },
        };

        const TypeMap3_2 = TypeMap3 + {
            "out1": {
                "context0": "contextstr0",
            },
            "out2": {
                "context1": 2,
                "name": "inp1",
            },
        };

        const TypeOpts3_1 = TypeOpts3 + {
            "runtime_keys": {
                # test auto cast from hash
                "context0": {
                    "unique_roles": "context",
                    "handler": auto sub (auto arg, hash<auto> ctx, *reference<bool> missing) {
                        return arg + "-ctx";
                    },
                },
                "context1": <MapperRuntimeKeyInfo>{
                    "unique_roles": "context",
                    "requires_roles": "value",
                    "value_type": Type::Int,
                    "handler": auto sub (int arg, auto val, hash<auto> ctx, *reference<bool> missing) {
                        return val + "-" + arg + "-ctx";
                    },
                },
            },
        };

        const TypeMap4 = TypeMap1 + {
            "output_2": {
                "name": "input_1",
                "type_options": {
                    "string.max_size_chars": 5,
                },
            },
        };

        const TypeOpts4 = TypeOpts1 + {
            "output": TypeOpts1.output + {
                "output_2": {
                    "desc": "output field 2",
                    "type": "string",
                },
            }
        };
    }

    constructor() : Test("MapperTest", "1.0") {
        addTestCase("bulk handler test", \bulkHandlerTest());
        addTestCase("struct test", \structTest());
        addTestCase("submapper test", \submapperTest());
        addTestCase("ignore_missing_input test", \ignoreMissingInputTest());
        addTestCase("use_input_record test", \useInputRecordTest());
        addTestCase("nullable output", \nullableOutputTest());
        addTestCase("list test", \listTest());
        addTestCase("runtime keys", \runtimeKeys());
        addTestCase("hash output", \hashOutputTest());
        addTestCase("test types", \testTypes());
        addTestCase("Test mapAll()", \testMapperMapAll());
        addTestCase("Test mapData()", \testMapperMapData());
        addTestCase("Test mapAuto()", \testMapperMapAuto());
        addTestCase("Runtime test", \testMapperRuntime());
        addTestCase("Test 'input' option", \testMapperOptionInput());
        addTestCase("Bulk Test", \testMapperBulk());
        addTestCase("Index test", \testMapperIndex());
        addTestCase("Empty String test", \testEmptyString());
        addTestCase("Test mapAll() with hash arg", \testMapperMapAllHash());
        addTestCase("Test mapFieldType", \testMapperMapFieldType());
        addTestCase("Dot test", \dotTest());
        addTestCase("field length test", \testFieldLength());
        set_return_value(main());
    }

    private {
        hash<auto> m_opts = {"runtime": {"runtimetest": 1}};
    }

    setUp() {
    }

    bulkHandlerTest() {
        hash<auto> mappings = {
            "out1": "in1",
            "out2": {
                "context": "str",
            },
        };
        Mapper mapper(mappings, {
            "runtime_keys": {
                "context": <MapperRuntimeKeyInfo>{
                    "handler": auto sub (auto arg, hash<auto> ctx, *reference<bool> missing) {
                        return arg + "-ctx";
                    },
                },
            },
        });

        hash<auto> input = {
            "in1": ("a", "b",),
            "in2": ("x", "y",),
        };
        hash<auto> expected_output = {"out1": ("a", "b"), "out2": ("str-ctx", "str-ctx",)};
        hash<auto> output = mapper.mapAuto(input);
        assertEq(expected_output, output);

        list<hash<auto>> output2 = mapper.mapAuto(new ListIterator(map $1, input.contextIterator()));
        assertEq((map $1, expected_output.contextIterator()), output2);
    }

    structTest() {
        hash<auto> mappings = {
            "out": {
                "struct": "a.b",
            },
        };
        Mapper mapper(mappings, {"input": {
            "a": {},
        }});
        hash<auto> input = {
            "a": {
                "b": 1,
            },
        };
        hash<auto> output = mapper.mapData(input);
        assertEq(1, output.out);
    }

    # issue #3414
    submapperTest() {
        assertThrows("MAP-ERROR", sub () { Mapper m({
            "f": {
                "submapper": "test",
            },
        },); });

        assertThrows("MAP-ERROR", sub () { Mapper m({
            "f": {
                "submapper_options": {},
            },
        },); });

        {
            code submapper_get_err = Mapper sub (string name, *hash<auto> opts) {
                throw "MAPPER-ERROR", sprintf("no such mapper %y", name);
            };

            Mapper m({
                "f": {
                    "submapper": "test",
                },
            }, {
                "submapper_get": submapper_get_err,
            });

            assertThrows("MAPPER-ERROR", \m.mapData(), {"a": 1});
        }

        {
            TestInputProvider input_provider();
            code log_input = sub (hash<auto> rec) {if (m_options.verbose > 2) { printf("input: %y\n", rec); }};
            code log_output = sub (hash<auto> rec) {if (m_options.verbose > 2) { printf("output: %y\n", rec); }};

            code submapper_get = Mapper sub (string name, *hash<auto> opts) {
                return new Mapper({
                    "sub-a": {
                        "name": "sub-a",
                    },
                    "sub-b": {
                        "name": "sub-b",
                    },
                }, {
                    "input_provider": input_provider,
                } + opts);
            };

            {
                Mapper m({
                    "f": {
                        "submapper": "test",
                        "submapper_options": {
                            "input_provider_search": {
                                "sub-a": "$local:input.a",
                            },
                        },
                        "submapper_auto_input": True,
                    },
                }, {
                    "submapper_get": submapper_get,
                    "input_log": log_input,
                    "output_log": log_output,
                });

                assertEq({"f": (
                    {"sub-a": "a-1", "sub-b": "b-1"},
                )}, m.mapData({"a": "a-1"}));

                assertEq({"sub-a": "a-1"}, input_provider.getLastWhereCond());
                assertEq({}, input_provider.getLastSearchOptions());
            }
        }
        {
            TestInputProvider input_provider();
            Mapper m({
                "sub-a": "a",
                "sub-b": "b",
            }, {
                "input_provider": input_provider,
                "input_suppress_data_provider": True,
            });
            assertThrows("MAPPER-INPUT-PROVIDER-ERROR", \m.mapAutoInput());
            assertThrows("MAPPER-INPUT-PROVIDER-ERROR", \m.runAutonomous());
            assertThrows("MAPPER-INPUT-PROVIDER-ERROR", \m.getOutputIterator());

            assertEq({"sub-a": "a-1", "sub-b": "b-1"}, m.mapData({"a": "a-1", "b": "b-1"}));
        }
        {
            TestInputProvider input_provider();
            TestOutputProvider output_provider();
            Mapper m({
                "sub-a": "a",
                "sub-b": "b",
            }, {
                "input_provider": input_provider,
                "output_provider": output_provider,
                "output_suppress_data_provider": True,
            });
            assertThrows("MAPPER-OUTPUT-PROVIDER-ERROR", \m.runAutonomous());
            assertThrows("MAPPER-OUTPUT-PROVIDER-ERROR", \m.commit());
            assertThrows("MAPPER-OUTPUT-PROVIDER-ERROR", \m.rollback());
            assertThrows("MAPPER-OUTPUT-BULK-ERROR", \m.flushOutput());
            assertThrows("MAPPER-OUTPUT-BULK-ERROR", \m.discardOutput());

            assertEq({"sub-a": "a-1", "sub-b": "b-1"}, m.mapData({"a": "a-1", "b": "b-1"}));
        }

        {
            code log_input = sub (hash<auto> rec) {if (m_options.verbose > 2) { printf("input: %y\n", rec); }};
            code log_output = sub (hash<auto> rec) {if (m_options.verbose > 2) { printf("output: %y\n", rec); }};

            code submapper_get = Mapper sub (string name, *hash<auto> opts) {
                return new Mapper({
                    "sub-a": {
                        "name": "sub-a",
                    },
                    "sub-b": {
                        "name": "sub-b",
                    },
                }, opts);
            };

            Mapper m({
                "_a": {
                    "name": "a",
                },
                "f": {
                    "name": "sub",
                    "submapper": "test",
                },
            }, {
                "submapper_get": submapper_get,
                "input_log": log_input,
                "output_log": log_output,
            });

            assertEq({
                "_a": "a-1",
                "f": (
                    {"sub-a": "a-0", "sub-b": "b-0"},
                    {"sub-a": "a-1", "sub-b": "b-1"},
                ),
            }, m.mapData({
                "a": "a-1",
                "sub": (
                    {"sub-a": "a-0", "sub-b": "b-0"},
                    {"sub-a": "a-1", "sub-b": "b-1"},
                ),
            }));
        }
    }

    # issue #3837
    ignoreMissingInputTest() {
        {
            Mapper m({
                "out0.x": "a",
                "out1": "a.b",
            }, {
                "ignore_missing_input": True,
                "output": {
                    "out0": new QoreDataField("out0", "test output field", HashOrNothingType),
                    "out1": new QoreDataField("out1", "test output field", StringOrNothingType),
                },
            });
            hash<auto> rec = {
                "x": 1,
                "a": {
                    "b": "one",
                },
            };
            assertEq({
                "out0": {
                    "x": rec.a,
                },
                "out1": rec.a.b,
            }, m.mapData(rec));

            rec.a = {"c": 2};
            assertEq({
                "out0": {
                    "x": rec.a,
                },
            }, m.mapData(rec));

            rec = {};
            assertEq({
                "out0": NOTHING,
            }, m.mapData(rec));
        }
    }

    # issue #3823
    useInputRecordTest() {
        {
            Mapper m({
                "out0": {
                    "use_input_record": True,
                },
                "out1": "a.b",
            }, {
                "output": {
                    "out0": new QoreDataField("out0", "test output field", HashType),
                    "out1": new QoreDataField("out1", "test output field", StringType),
                },
            });
            hash<auto> rec = {
                "x": 1,
                "a": {
                    "b": "one",
                },
            };
            assertEq({
                "out0": rec,
                "out1": rec.a.b,
            }, m.mapData(rec));
        }

        {
            Mapper m({
                "out0": {
                    "use_input_record": True,
                },
                "out1": "a.b",
            }, {
                "output": {
                    "out0": new QoreDataField("out0", "test output field", HashType),
                    "out1": new QoreDataField("out1", "test output field", SoftStringType),
                },
            });
            hash<auto> rec = {
                "x": 1,
                "a": {
                    "b": "one",
                },
            };
            assertEq({
                "out0": rec,
                "out1": rec.a.b,
            }, m.mapData(rec));
        }

        assertThrows("MAP-ERROR", sub () {
            Mapper m({
                "out0": {
                    "use_input_record": True,
                },
                "out1": "a.b",
            }, {
                "output": {
                    "out0": new QoreDataField("out0", "test output field", StringType),
                    "out1": new QoreDataField("out1", "test output field", StringType),
                },
            });
        });
    }

    # issue #3788
    nullableOutputTest() {
        {
            Mapper m({"out0": "in0"}, {
                "output": {"out0": new QoreDataField("out0", "test output field", StringType)},
            });
            assertEq("str", m.mapData({"in0": "str"}).out0);
        }
        {
            Mapper m({"out0": "in0"}, {
                "output": {"out0": new QoreDataField("out0", "test output field", StringType)},
            });
            assertThrows("RUNTIME-TYPE-ERROR", \m.mapData(), {"inp0": NOTHING});
        }
        {
            Mapper m({"out0": "in0"}, {
                "output": {"out0": new QoreDataField("out0", "test output field", StringType)},
            });
            m.setNullableOutput();
            assertNothing(m.mapData({"in0": NOTHING}).out0);
        }
        {
            Mapper m({"out0": "in0"}, {
                "output": {"out0": new QoreDataField("out0", "test output field", StringType)},
                "output_nullable": True,
            });
            assertNothing(m.mapData({"in0": NOTHING}).out0);
        }
        HashDataType h0();
        h0.addField(new QoreDataField("id", "", IntType));
        HashDataType h1();
        h1.addField(new QoreDataField("rec", "", h0));
        {
            Mapper m({"out0.rec.id": {"name": "in0"}}, {
                "output": {"out0": new QoreDataField("out0", "test output field", h1)},
            });
            assertEq(1, m.mapData({"in0": 1}).out0.rec.id);
        }
        {
            Mapper m({"out0.rec.id": {"name": "in0"}}, {
                "output": {"out0": new QoreDataField("out0", "test output field", h1)},
                "output_nullable": True,
            });
            assertNothing(m.mapData({"in0": NOTHING}).out0.rec.id);
        }
    }

    listTest() {
        Mapper m(TypeMap0, TypeOpts0);
        assertEq((1, 2), m.mapData({"input_1": (1, 2)}).output_1);
    }

    runtimeKeys() {
        {
            Mapper m(TypeMap3, TypeOpts3);
            hash<auto> h = m.mapData({"inp0": "input-0", "inp1": "input-1"});
            assertEq({
                "out0": "input-0",
                "out1": "contextstr0-ctx",
                "out2": "input-1-contextstr1-ctx",
            }, h);
        }
        assertThrows("MAP-ERROR", "context1.*expected type", sub () { Mapper m(TypeMap3, TypeOpts3_1); });
        assertThrows("MAP-ERROR", "key.*role", sub () { Mapper m(TypeMap3_1, TypeOpts3_1); });
        {
            Mapper m(TypeMap3_2, TypeOpts3_1);
            hash<auto> h = m.mapData({"inp0": "input-0", "inp1": "input-1"});
            assertEq({
                "out0": "input-0",
                "out1": "contextstr0-ctx",
                "out2": "input-1-2-ctx",
            }, h);
        }
    }

    hashOutputTest() {
        Mapper m(TypeMap2, TypeOpts2);
        hash<auto> h = m.mapData({"input_0": {"k1": 1, "k2": 2}, "input_1": "one"});
        assertEq({
            "output_0": {
                "key1": 1,
                "key2": 2,
            },
            "output_1": "one",
        }, h);
    }

    testTypes() {
        {
            hash<auto> map_desc = {
                "input": {
                    "input": {
                        "desc": "input field",
                        "type": "any",
                    },
                },
                "output": (new TestHashType()).getFields(),
            };

            Mapper m({"output.stringval": {"name": "input"}}, map_desc);
            assertEq("1", m.mapData({"input": 1}).output.stringval);
        }
        {
            SomethingDataType something();
            assertEq(1, something.acceptsValue(1));
            assertTrue(something.isAssignableFrom(IntType));
            assertFalse(something.isAssignableFrom(new Type("nothing")));
        }
        {
            Mapper m(TypeMap0, TypeOpts0);
            assertEq("x", m.mapData({"input_1": "x"}).output_1);
            assertEq(1n, m.mapData({"input_1": 1n}).output_1);
            auto v = m.mapData({"input_1": <StatInfo>{}}).output_1;
            assertEq(<StatInfo>{}, v);
            assertEq("hash<StatInfo>", v.fullType());

            hash<auto> x = {"a": get_random_string(), "b": get_random_bytes(20)};
            assertEq(x, m.mapData({"input_0": x}).output_0);
            v = m.mapData({"input_0": <StatInfo>{}}).output_0;
            assertEq(<StatInfo>{}, v);
            assertEq("hash<StatInfo>", v.fullType());
            assertThrows("RUNTIME-TYPE-ERROR", \m.mapData(), {"input_0": 1});
        }
        {
            Mapper m(TypeMap1, TypeOpts1);
            assertEq("x", m.mapData({"input_1": "x"}).output_1);
            assertEq(1n, m.mapData({"input_1": 1n}).output_1);
            auto v = m.mapData({"input_1": <StatInfo>{}}).output_1;
            assertEq(<StatInfo>{}, v);
            assertEq("hash<StatInfo>", v.fullType());

            hash<auto> x = {"a": get_random_string(), "b": get_random_bytes(20)};
            assertEq(x, m.mapData({"input_0": x}).output_0);
            v = m.mapData({"input_0": <StatInfo>{}}).output_0;
            assertEq(<StatInfo>{}, v);
            assertEq("hash<StatInfo>", v.fullType());
            assertThrows("RUNTIME-TYPE-ERROR", \m.mapData(), {"input_0": 1});
        }

        {
            Mapper m(TypeMap4, TypeOpts4);
            assertEq("x", m.mapData({"input_1": "x"}).output_2);
            assertThrows("STRING-TOO-LONG", \m.mapData(), {"input_1": "123456"});
        }
    }

    testMapperMapAll() {
        Mapper m(DataMap, m_opts);
        list l = m.mapAll(MapInput);
        testAssertion("Verify mapped list", \equalsIterated(), (new ListIterator(l), new ListIterator(MapOutput)));
        testAssertion("Verify item count", \equals(), (m.getCount(), 2));
    }

    testMapperMapData() {
        Mapper m(DataMap, m_opts);
        list l = map m.mapData($1), MapInput;
        testAssertion("Verify mapped list", \equalsIterated(), (l.iterator(), MapOutput.iterator()));
        testAssertion("Verify item count", \equals(), (m.getCount(), 2));
    }

    testMapperMapAuto() {
        # test list of hashes
        Mapper m1(DataMap, m_opts);
        assertNothing(m1.mapAuto(NOTHING));
        auto res = m1.mapAuto(MapInput);
        testAssertion("Verify mapped list", \equalsIterated(), (MapOutput.iterator(), res.iterator()));
        testAssertion("Verify item count", \equals(), (m1.getCount(), 2));

        # test hash of lists with disabled bulk
        Mapper m2(DataMap, m_opts + {"disable_bulk": True});
        res = m2.mapAuto(MapInput2);
        testAssertion("Verify mapped hash of lists", \equalsIterated(), (MapOutput.iterator(), res.iterator()));
        testAssertion("Verify item count", \equals(), (m2.getCount(), 2));

        # test hash of lists with enabled bulk
        Mapper m3(DataMap2, m_opts + {
            "output": {
                "int_field": {},
                "date_field": {
                    "type": "date",
                },
            },
            "global_type_options": {
                "date.format": "DD.MM.YYYY",
            }
        });
        res = m3.mapAuto(MapInput3);
        testAssertion("Verify mapped hash of lists", \equalsIterated(), (MapOutput2.iterator(), res.contextIterator()));
        testAssertion("Verify item count", \equals(), (m3.getCount(), 3));

        # test one record hash
        Mapper m4(TypeMap2, TypeOpts2);
        res = m4.mapAuto({"input_0": {"k1": 213, "k2": 453}, "input_1": "sdfj"});
        assertEq({
            "output_0": {
                "key1": 213,
                "key2": 453,
            },
            "output_1": "sdfj",
        }, res);

        # test invalid input type
        assertThrows("MAPPER-INPUT-TYPE-ERROR", \m4.mapAuto(), "asdsadmn");

        # issue #3931: test with bulk APIs and context
        Mapper m5(DataMap2 + {
            "ctx0": {
                "context0": "c0",
            },
            "ctx1": {
                "name": "count",
                "context1": "c1",
            },
            "ctx2": {
                "constant": "x",
            },
        }, {
            "output": {
                "int_field": {},
                "date_field": {
                    "type": "date",
                },
                "ctx0": {
                    "type": "string",
                },
                "ctx1": {
                    "type": "string",
                },
                "ctx2": {
                    "type": "string",
                },
            },
            "global_type_options": {
                "date.format": "DD.MM.YYYY",
            },
            "runtime_keys": {
                "context0": <MapperRuntimeKeyInfo>{
                    "handler": auto sub (auto arg, hash<auto> ctx, *reference<bool> missing) {
                        return arg + "-ctx";
                    },
                },
                "context1": <MapperRuntimeKeyInfo>{
                    "requires_roles": "value",
                    "handler": auto sub (auto arg, auto val, hash<auto> ctx, *reference<bool> missing) {
                        return val + "-" + arg + "-ctx";
                    },
                },
            },
        });
        res = m5.mapAuto({"count": 11, "idate": "10.12.2019"});
        assertEq({
            "int_field": 11,
            "date_field": 2019-12-10,
            "ctx0": "c0-ctx",
            "ctx1": "11-c1-ctx",
            "ctx2": "x",
        }, res);
        res = m5.mapAuto(MapInput3);
        assertEq({
            "int_field": (11, 10, NOTHING),
            "date_field": (2019-12-10, 2019-12-11, 2019-12-12),
            "ctx0": ("c0-ctx", "c0-ctx", "c0-ctx"),
            "ctx1": ("11-c1-ctx", "10-c1-ctx", "-c1-ctx"),
            "ctx2": "x",
        }, res);
    }

    testMapperRuntime() {
        Mapper m(DataMap, m_opts);
        ListIterator it(MapInput);
        int ix = 0;
        m.setRuntime("runtimetest", 0);
        while (it.next()) {
            hash rec = m.mapData(it.getValue());
            hash orig = MapOutput[ix] + ("runtimetest" : 0);
            testAssertion(sprintf("runtime item: %d", 0), \equals(), (rec, orig));
            ix++;
        }
    }

    testMapperOptionInput() {
        hash my_opts = m_opts + ("input": map {$1: $1}, MapInput[0].keys());
        Mapper m(DataMap, my_opts);
        hash<auto> h = m.mapData(MapInput[0]);
        assertEq(1, h.id);
    }

    testMapperBulk() {
        MapperIterator i(MapInput.iterator(), DataMap, m_opts);
        list l = i.mapBulk(MapInput.lsize());
        testAssertion("Verify mapped list", \equalsIterated(), (l.iterator(), MapOutput.iterator()));
        assertEq(2, i.getCount());
    }

    testMapperIndex() {
        Mapper m(DataMap, m_opts);
        list l = m.mapAll(MapInput);
        assertEq(l, MapOutput);

        Mapper mi(DataMap, m_opts);
        ListIterator it(MapInput);
        int ix = 0;
        while (it.next()) {
            hash rec = mi.mapData(it.getValue());
            assertEq(rec, MapOutput[ix]);
            ix++;
        }
    }

    testEmptyString() {
        Mapper m(DataMap, m_opts + ("empty_strings_to_nothing": True));
        list l = m.mapAll(MapInput);
        list o = MapOutput;
        # simulate "old empty string to NOTHING conversion"
        o[0].empty_string = NOTHING;
        o[1].empty_string = NOTHING;
        assertEq(l, o);
    }

    testFieldLength() {
        Mapper m(DataMap, m_opts);
        list input = MapInput;
        input[0].length_test = "longer string";
        # test issue #2495
        assertThrows("STRING-TOO-LONG", "length_test", \m.mapAll(), (input,));
    }

    testMapperMapAllHash() {
        Mapper m(DataMap, m_opts);
        list l = m.mapAll(MapInput2);
        assertEq(MapOutput, l);
    }

    testMapperMapFieldType() {
        Mapper m(DataMap2, m_opts + {
            "output": {
                "int_field": {},
                "date_field": {
                    "type": "date",
                },
            },
            "global_type_options": {
                "date.format": "DD.MM.YYYY",
            }
        });
        list<auto> l = m.mapAll(MapInput3);
        assertEq(MapOutput2, l);
    }

    dotTest() {
        {
            Mapper m(DotMap, {"allow_dot": True, "allow_output_dot": True});
            list l = m.mapAll(DotInput);
            assertEq(DotOutput, l);
            assertEq(2, m.getCount());

            # test for hash of lists format
            hash h = map {$1: ()}, DotInput[0].keyIterator();
            map (map h{$1.key} += $1.value, $1.pairIterator()), DotInput;
            assertEq(DotOutput, m.mapAll(h));
            assertEq(4, m.getCount());
        }

        {
            Mapper m(DotMap2);
            list<auto> l = m.mapAll(DotInput);
            assertEq(DotOutput, l);
            assertEq(2, m.getCount());

            # test for hash of lists format
            hash<string, list<auto>> h = map {$1: ()}, keys DotInput[0];
            map (map h{$1.key} += $1.value, $1.pairIterator()), DotInput;
            assertEq(DotOutput, m.mapAll(h));
            assertEq(4, m.getCount());
        }

        {
            HashDataType input1();
            input1.addField(new QoreDataField("input1-1", "", IntType));

            hash<string, AbstractDataField> input = {
                "input1": new QoreDataField("input1", "", input1),
                "input1.input1-1": new QoreDataField("input1.input1-1", "", IntType),
                "input1.input1-2": new QoreDataField("input1.input1-2", "", IntType),
                "a": new QoreDataField("a", "", HashType),
            };

            HashDataType output1();
            output1.addField(new QoreDataField("output1-1", "", SoftStringType));

            hash<string, AbstractDataField> output = {
                "output1": new QoreDataField("output1", "", output1),
                "output1.output1-1": new QoreDataField("output1.output1-1", "", SoftStringType),
                "output.2": new QoreDataField("output.2", "", SoftIntType),
                "output.3": new QoreDataField("output.3", "", IntType),
            };

            hash<auto> map_config = {
                "output1.output1-1": "input1\\.input1-2",
                "output1\\.output1-1": "input1\\.input1-1",
                "output\\.2": "input1.input1-1",
                "output\\.3": "a.b",
            };

            list<hash<auto>> input_data = (
                {"input1": {"input1-1": 1}, "input1.input1-1": 2, "input1.input1-2": 3, "a": {"b": 4}},
                {"input1": {"input1-1": 5}, "input1.input1-1": 6, "input1.input1-2": 7, "a": {"b": 8}},
            );

            list<hash<auto>> output_data = (
                {"output1": {"output1-1": "3"}, "output1.output1-1": "2", "output.2": 1, "output.3": 4},
                {"output1": {"output1-1": "7"}, "output1.output1-1": "6", "output.2": 5, "output.3": 8},
            );

            Mapper m(map_config, {"input": input, "output": output});
            list<auto> l = m.mapAll(input_data);
            assertEq(output_data, l);
            assertEq(2, m.getCount());

            # test for hash of lists format
            hash<string, list<auto>> h = map {$1: ()}, keys input_data[0];
            map (map h{$1.key} += $1.value, $1.pairIterator()), input_data;
            assertEq(output_data, m.mapAll(h));
            assertEq(4, m.getCount());
        }

        {
            Mapper m(OStructMap, ("allow_dot": True,));
            list l = m.mapAll(OStructInput);
            assertEq(OStructOutput, l);
            assertEq(2, m.getCount());

            # test for hash of lists format
            hash h = map {$1: ()}, OStructInput[0].keyIterator();
            map (map h{$1.key} += $1.value, $1.pairIterator()), OStructInput;
            assertEq(OStructOutput, m.mapAll(h));
            assertEq(4, m.getCount());
        }
    }
}
