#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

%new-style
%enable-all-warnings
%require-types
%strict-args

%requires ../../../../qlib/Util.qm
%requires ../../../../qlib/QUnit.qm
%requires ../../../../qlib/DataProvider
%requires ../../../../qlib/CsvUtil

%exec-class CsvUtilTest

public class CsvUtilTest inherits QUnit::Test {
    private {
        const CsvInput = "UK,1234567890,\"Sony, Xperia S\",31052012,true
UK,1234567891,\"Sony, Xperia S\",31052012,true
UK,1234567892,\"Sony, Xperia S\",31052012,false
UK,1234567893,\"Sony, Xperia S\",31052012,true\n";

        const CsvInputWithHeader = "CC,serno,desc,received,activated\n" + CsvInput;

        const CsvRecords = (
            ("cc": "UK", "serno": 1234567890, "desc": "Sony, Xperia S", "received": 2012-05-31, "activated": True),
            ("cc": "UK", "serno": 1234567891, "desc": "Sony, Xperia S", "received": 2012-05-31, "activated": True),
            ("cc": "UK", "serno": 1234567892, "desc": "Sony, Xperia S", "received": 2012-05-31, "activated": False),
            ("cc": "UK", "serno": 1234567893, "desc": "Sony, Xperia S", "received": 2012-05-31, "activated": True),
        );
        const CsvMultiInput = "1,2016-01,20160401
2,123,\"Sony, Xperia S\",1,100
2,124,\"Nokia, Lumia\",2,150
3,250
";

        const CsvMultiRecords = (
            ("type": "header", "record": ("type": 1, "invoice_no": "2016-01", "date": 2016-04-01)),
            ("type": "item", "record": ("type": 2, "item_no": 123, "item": "Sony, Xperia S", "pcs": 1, "price": 100)),
            ("type": "item", "record": ("type": 2, "item_no": 124, "item": "Nokia, Lumia", "pcs": 2, "price": 150)),
            ("type": "footer", "record": ("type": 3, "total": 250)),
        );

        const CsvIntInput = "f1:2,03:3:4,05
f2:3,00:5:25,34";

        const CsvNumOutput = (
            {
                "label": "f1",
                "amount1": 2.03,
                "amount2": 3,
                "amount3": 4.05n,
            },
            {
                "label": "f2",
                "amount1": 3.0,
                "amount2": 5,
                "amount3": 25.34n,
            },
        );
    }

    constructor() : Test("CsvUtilTest", "1.0") {
        addTestCase("data provider", \dataProviderTest());
        addTestCase("multi-line fields", \multiLineFieldTest());
        addTestCase("issue 2806 test", \issue2806());
        addTestCase("issue 2746 test", \issue2746());
        addTestCase("Basic CSV tests", \csvTest());
        addTestCase("Optional field CSV tests", \csvOptionalFieldTest());
        addTestCase("Multi-type CSV tests", \csvMultiTest());
        addTestCase("Old style CSV tests", \csvOldStyleMultiTest());
        addTestCase("CsvUtil non-numeric values", \csvNonNumeric());
        addTestCase("Escaping", \escapeTest());
        addTestCase("csvData test", \csvData());
        addTestCase("Single-type CSV tests", \csvSingleTest());
        addTestCase("Single-type CSV map tests", \csvSingleMapTest());
        addTestCase("Multi-type resolve CSV tests", \csvMultiResolveTest());
        addTestCase("Config stress", \csvConfig());
        addTestCase("CSV file iterator", \csvFileIterator());
        addTestCase("issue 2739 test", \issue2739());

        # Return for compatibility with test harness that checks return value.
        set_return_value(main());
    }

    multiLineFieldTest() {
        hash<auto> opts = {
            "header-names": True,
        };

        {
            string csvdata = "A,B\nfield 1,\"field 2\nmulti-line\"\nl2,l3\n";

            CsvDataIterator i(csvdata, opts);
            list<auto> l = map $1, i;
            assertEq((
                {
                    "A": "field 1",
                    "B": "field 2\nmulti-line",
                },
                {
                    "A": "l2",
                    "B": "l3",
                },
            ), l);
        }

        {
            string csvdata = "A,B\n\"field 1\nline 2\nline 3\",\"field 2\nmulti-line\"\nl2,l3\n";

            CsvDataIterator i(csvdata, opts);
            list<auto> l = map $1, i;
            assertEq((
                {
                    "A": "field 1\nline 2\nline 3",
                    "B": "field 2\nmulti-line",
                },
                {
                    "A": "l2",
                    "B": "l3",
                },
            ), l);
        }
    }

    dataProviderTest() {
        list<hash<auto>> rows;
        {
            CsvReadDataProvider dp;

            hash<auto> constructor_options = {
                "header_lines": 1,
                "header_names": True,
                "date_format": "DDMMYYYY",
                "fields": {
                    "received": {
                        "type": "date",
                    },
                    "activated": "bool",
                },
            };

            {
                dp = DataProvider::getFactoryObjectWithTemplate("csvread", {
                    "path": "$template:value",
                }, constructor_options);
                hash<DataProviderInfo> info = dp.getInfo();
                assertEq(Type::String, info.name.type());
                assertTrue(info.has_record);
                assertTrue(info.supports_read);
                assertEq(Type::String, info.desc.type());
            }

            {
                dp = DataProvider::getFactoryObjectFromExample("csvread", CsvInputWithHeader, constructor_options);
                hash<DataProviderInfo> info = dp.getInfo();
                assertEq(Type::String, info.name.type());
                assertTrue(info.has_record);
                assertTrue(info.supports_read);
                assertEq(Type::String, info.desc.type());
                rows = map $1, dp.searchRecords();
                assertEq(5, rows[0].size());
                assertEq(4, rows.size());
            }

            {
                dp = DataProvider::getFactoryObjectFromExample("csvread", new StringInputStream(CsvInputWithHeader),
                    constructor_options);
                hash<DataProviderInfo> info = dp.getInfo();
                assertEq(Type::String, info.name.type());
                assertTrue(info.has_record);
                assertTrue(info.supports_read);
                assertEq(Type::String, info.desc.type());
                rows = map $1, dp.searchRecords();
                assertEq(5, rows[0].size());
                assertEq(4, rows.size());
            }

            # the CSV data can only be iterated once and then the provider needs to be recreated
            code reset = sub () {
                dp = new CsvReadDataProvider(new StringInputStream(CsvInputWithHeader), constructor_options);
            };

            {
                AbstractDataProviderFactory factory = DataProvider::getFactoryEx("csvread");
                hash<DataProviderFactoryInfo> info = factory.getInfo();
                assertEq("csvread", info.name);
                assertFalse(info.api_management);
                assertEq(Type::String, info.desc.type());
                rows = factory.getExampleProviderRecordOutput(new StringInputStream(CsvInputWithHeader), constructor_options);
                assertEq(5, rows[0].size());
                assertEq(4, rows.size());

                AbstractDataProvider dp0 = factory.getProviderFromExample(
                    new StringInputStream(CsvInputWithHeader), constructor_options
                );
                rows = map $1, dp0.searchRecords();
                assertEq(5, rows[0].size());
                assertEq(4, rows.size());
            }

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertFalse(info.supports_bulk_read);
            }

            {
                reset();
                hash<DataProviderExpression> cond = <DataProviderExpression>{
                    "exp": DP_OP_OR,
                    "args": (
                        <DataProviderExpression>{
                            "exp": DP_SEARCH_OP_EQ,
                            "args": (
                                <DataProviderFieldReference>{"field": "serno"},
                                1234567892,
                            ),
                        },
                        <DataProviderExpression>{
                            "exp": DP_SEARCH_OP_EQ,
                            "args": (
                                <DataProviderFieldReference>{"field": "serno"},
                                1234567893,
                            ),
                        },
                    )
                };
                rows = map $1, dp.searchRecords(cond);
                assertEq(5, rows[0].size());
                assertEq(2, rows.size());

                reset();
                cond.args += <DataProviderExpression>{
                    "exp": DP_OP_AND,
                    "args": (
                        <DataProviderExpression>{
                            "exp": DP_SEARCH_OP_NE,
                            "args": (
                                <DataProviderFieldReference>{"field": "activated"},
                                False,
                            ),
                        },
                        <DataProviderExpression>{
                            "exp": DP_SEARCH_OP_GE,
                            "args": (
                                <DataProviderFieldReference>{"field": "received"},
                                2012-05-31,
                            ),
                        },
                    ),
                };
                rows = map $1, dp.searchRecords(cond);
                assertEq(5, rows[0].size());
                assertEq(4, rows.size());

                reset();
                cond.args[0].args[0].field = "xyz";
                assertThrows("INVALID-OPERATION", sub () { map $1, dp.searchRecords(cond); });
            }

            {
                reset();
                rows = map $1, dp.searchRecords();
                assertEq(5, rows[0].size());
                assertEq(4, rows.size());

                reset();
                *hash<auto> row = dp.searchSingleRecord({"serno": "1234567892"});
                assertEq(Type::Hash, row.type());
                assertEq("1234567892", row.serno);
                assertEq(Type::String, row.serno.type());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({}, {"columns": ("CC", "serno")});
                assertEq(2, rows[0].size());
                assertEq(4, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({}, {"limit": 1});
                assertEq(5, rows[0].size());
                assertEq(1, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({}, {"limit": 1, "offset": 1});
                assertEq(5, rows[0].size());
                assertEq(1, rows.size());
                hash<auto> row1 = shift rows;

                reset();
                rows = map $1, dp.searchRecords({}, {"limit": 1, "offset": 2});
                assertEq(5, rows[0].size());
                assertEq(1, rows.size());
                hash<auto> row2 = shift rows;

                reset();
                rows = map $1, dp.searchRecords();
                assertEq(rows[1], row1);
                assertEq(rows[2], row2);
            }

            {
                reset();
                AbstractDataProviderBulkRecordInterface i = dp.getBulkRecordInterface();
                hash<auto> q = i.getValue();
                assertEq(Type::List, q.firstValue().type());
                assertEq(4, q.firstValue().lsize());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"serno": dp_op_eq("1234567892")});
                assertEq(1, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"serno": dp_op_ne("1234567892")});
                assertEq(3, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"serno": dp_op_ne()});
                assertEq(4, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"serno": dp_op_like("%93")});
                assertEq(1, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"serno": dp_op_regex("93$")});
                assertEq(1, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"received": dp_op_ge(2012-05-31)});
                assertEq(4, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"received": dp_op_gt(2012-05-31)});
                assertEq(0, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"received": dp_op_le(2012-05-31)});
                assertEq(4, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"received": dp_op_lt(2012-05-31)});
                assertEq(0, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"CC": dp_op_in("US", "IE")});
                assertEq(0, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"CC": dp_op_in("UK", "UA")});
                assertEq(4, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"CC": dp_op_between("UJ", "UL")});
                assertEq(4, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"CC": dp_op_between("AA", "AX")});
                assertEq(0, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({"serno": dp_op_not(dp_op_like("%93"))});
                assertEq(3, rows.size());
            }
        }

        {
            CsvReadDataProvider dp;

            # the CSV data can only be iterated once and then the provide needs to be recreated
            code reset = sub () {
                dp = new CsvReadDataProvider({
                    "stream": new StringInputStream(CsvInputWithHeader),
                    "header_names": True,
                    "date_format": "DDMMYYYY",
                    "fields": {
                        "received": {
                            "type": "date",
                        },
                        "activated": "bool",
                    },
                });
            };

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertFalse(info.supports_bulk_read);
            }

            {
                reset();
                rows = map $1, dp.searchRecords();
                assertEq(4, rows.size());

                reset();
                *hash<auto> row = dp.searchSingleRecord({"serno": "1234567892"});
                assertEq(Type::Hash, row.type());
                assertEq("1234567892", row.serno);
                assertEq(Type::String, row.serno.type());
            }

            {
                reset();
                AbstractDataProviderBulkRecordInterface i = dp.getBulkRecordInterface();
                hash<auto> q = i.getValue();
                assertEq(Type::List, q.firstValue().type());
                assertEq(4, q.firstValue().lsize());
            }

        }

        {
            CsvWriteDataProvider dp;
            StringOutputStream stream;

            # the CSV data can only be iterated once and then the provide needs to be recreated
            code reset = sub () {
                stream = new StringOutputStream();
                dp = new CsvWriteDataProvider(stream, {
                    "write_headers": False,
                    "date_format": "DDMMYYYY",
                    "fields": {
                        "CC": {
                            "type": "string",
                        },
                        "serno": {
                            "type": "int",
                        },
                        "desc": {
                            "type": "string",
                        },
                        "received": {
                            "type": "date",
                        },
                        "activated": "bool",
                    },
                });
            };

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertTrue(info.supports_create);
                assertFalse(info.supports_bulk_read);

                map dp.createRecord($1), rows;

                assertEq(CsvInput, stream.getData());
            }
        }

        {
            CsvWriteDataProvider dp;
            StringOutputStream stream;

            # the CSV data can only be iterated once and then the provide needs to be recreated
            code reset = sub () {
                stream = new StringOutputStream();
                dp = new CsvWriteDataProvider({
                    "stream": stream,
                    "write_headers": False,
                    "date_format": "DDMMYYYY",
                    "fields": {
                        "CC": {
                            "type": "string",
                        },
                        "serno": {
                            "type": "int",
                        },
                        "desc": {
                            "type": "string",
                        },
                        "received": {
                            "type": "date",
                        },
                        "activated": "bool",
                    },
                });
            };

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertTrue(info.supports_create);
                assertFalse(info.supports_bulk_read);

                map dp.createRecord($1), rows;

                assertEq(CsvInput, stream.getData());
            }
        }
    }

    issue2806() {
        hash<auto> opts = {
            "fields": {
                "label": "string",
                "amount1": "float",
                "amount2": "int",
                "amount3": "number",
            },
            "number_format": ".,",
            "separator": ":",
        };

        {
            CsvDataIterator i(CsvIntInput, opts);
            list<auto> l = map $1, i;
            assertEq(CsvNumOutput, l);

            CsvStringWriter w({"number_format": ".,3"});
            string str = w.write((l[0].values(),));
            assertEq("f1,\"2,030\",\"3,000\",\"4,050\"\n", str);
        }
    }

    issue2746() {
        hash<auto> opts = {
            "header-names": False,
            "headers": ("CC", "serno", "desc", "received", "activated", "na1",),
        };

        {
            CsvDataIterator i(CsvInput, opts);
            list<auto> l = map $1.na1, i;
            assertEq((NOTHING, NOTHING, NOTHING, NOTHING, ), l);
        }

        {
            CsvDataIterator i(CsvInput, opts + {"compat_force_empty_string": True});
            list<auto> l = map $1.na1, i;
            assertEq(("", "", "", "",), l);
        }

        {
            csvutil_set_global_compat_force_empty_string(True);
            CsvDataIterator i(CsvInput, opts);
            list<auto> l = map $1.na1, i;
            assertEq(("", "", "", "",), l);
        }

        {
            csvutil_set_global_compat_force_empty_string(False);
            CsvDataIterator i(CsvInput, opts);
            list<auto> l = map $1.na1, i;
            assertEq((NOTHING, NOTHING, NOTHING, NOTHING, ), l);
        }
    }

    csvTest() {
        hash<auto> opts = {
            "fields": {
                "cc": "string",
                "serno": "int",
                "desc": "string",
                "received": {
                    "type": "date",
                    "format": "DDMMYYYY",
                },
                "activated": "bool",
            },
        };

        # Iterate through source
        AbstractCsvIterator i = new CsvDataIterator(CsvInput, opts);
        #list l = map $1, i;
        testAssertion("CsvDataIterator 1", \equalsIterated(), (new ListIterator(CsvRecords), i));

        CsvStringWriter w(("write-headers": False, "optimal-quotes" : True, "quote_escape" : '"') + opts);
        string outstr = w.write(CsvRecords);
        testAssertion("CsvDataWriter writer", \equals(), (CsvInput, outstr));  # ending CR is assymetrical when empty lines are not ignored

        # test with empty data and header lines
        i = new CsvDataIterator("", ("header-lines": 1));
        testAssertion("CsvDataIterator 2", \i.next(), (), RESULT_FAILURE);

        i = new CsvDataIterator("CaMeL,lilliput,SHREK\nRoW1,row1,ROW1\n", ("header-lines": 1, "header_names": True, "tolwr": True));
        testAssertionValue("CsvDataIterator lwr 1", i.index(), 0);
        testAssertionValue("CsvDataIterator lwr 1", i.lineNumber(), 0);
        testAssertionValue("CsvDataIterator lwr 1", i.next(), True);
        testAssertionValue("CsvDataIterator lwr 1", i.index(), 1);
        testAssertionValue("CsvDataIterator lwr 1", i.lineNumber(), 2);
        testAssertionValue("CsvDataIterator lwr 1 member", (i.camel, i.lilliput, i.shrek), ("RoW1", "row1", "ROW1"));
        assertThrows("ABSTRACTCSVITERATOR-FIELD-ERROR", CsvDataIterator sub() {return i.fiona;}, (), NOTHING);

        testAssertion("CsvDataIterator lwr 2", \equals(), (("camel": "RoW1", "lilliput": "row1", "shrek": "ROW1"), i.getValue()));
        testAssertionValue("CsvDataIterator lwr 3", i.getHeaders(), ("camel", "lilliput", "shrek"));
        testAssertionValue("CsvDataIterator lwr 4", i.next(), False);

        i = new CsvDataIterator("CaMeL,lilliput,SHREK\nRoW1,row1,ROW1\n", ("header-lines": 1, "header_names": True, ));
        testAssertionValue("CsvDataIterator !lwr 1", i.next(), True);
        testAssertion("CsvDataIterator !lwr 2", \equals(), (("CaMeL": "RoW1", "lilliput": "row1", "SHREK": "ROW1"), i.getValue()));
        testAssertionValue("CsvDataIterator !lwr 3", i.getHeaders(), ("CaMeL", "lilliput", "SHREK"));

        i = new CsvDataIterator("CaMeL,lilliput,SHREK\nRoW1,row1,ROW1\n", ("header-lines": 0, "header_names": False, "tolwr": True, "headers": ("CaMeL","lilliput","SHREK")));
        testAssertionValue("CsvDataIterator lwr h 1", i.getHeaders(), ("camel", "lilliput", "shrek"));

        i = new CsvDataIterator("row1-1,row1-2,row1-3\n", ("header-lines": 0, "header_names": False, ));
        testAssertionValue("CsvDataIterator header by idx 1", i.next(), True);
        testAssertion("CsvDataIterator header by idx 2", \equals(), (("0": "row1-1", "1": "row1-2", "2": "row1-3"), i.getValue()));
        testAssertion("CsvDataIterator header by idx 3", \equals(), (("row1-1", "row1-2", "row1-3"), i.getRecordList()));
        testAssertionValue("CsvDataIterator header by idx 4", i.getHeaders(), ("0", "1", "2"));

        i = new CsvIterator(new StringInputStream("row1-1,row1-2,row1-3\n"), NOTHING, ("header-lines": 0, "header_names": False, ));
        testAssertionValue("CsvDataIterator header by idx 1", i.next(), True);
        testAssertion("CsvDataIterator header by idx 2", \equals(), (("0": "row1-1", "1": "row1-2", "2": "row1-3"), i.getValue()));
        testAssertion("CsvDataIterator header by idx 3", \equals(), (("row1-1", "row1-2", "row1-3"), i.getRecordList()));
        testAssertionValue("CsvDataIterator header by idx 4", i.getHeaders(), ("0", "1", "2"));
    }

    csvOptionalFieldTest() {
        hash<auto> opts = ("fields": ("f1": "string", "f2": "int", "f3": ("type": "date", "format": "DDMMYYYY"), "f4": "*string", "f5": "*string"));
        string input = "UK,1234567890,31052012,O1-4
UK,1234567891,30052012,,O2-5
UK,1234567892,29052012,O3-4
UK,1234567893,28052012";

        list<auto> records = (
            ("f1": "UK", "f2": 1234567890, "f3": 2012-05-31, "f4": "O1-4", "f5": NOTHING),
            ("f1": "UK", "f2": 1234567891, "f3": 2012-05-30, "f4": NOTHING, "f5": "O2-5"),
            ("f1": "UK", "f2": 1234567892, "f3": 2012-05-29, "f4": "O3-4", "f5": NOTHING),
            ("f1": "UK", "f2": 1234567893, "f3": 2012-05-28, "f4": NOTHING, "f5": NOTHING),
        );

        # Iterate through source
        CsvDataIterator i(input, opts);
        #list l = map $1, i;
        testAssertion("CsvDataIterator optional fields", \equalsIterated(), (new ListIterator(records), i));
    }

    csvMultiTest() {
        hash<auto> spec = (
            "header": (
                "type": ("type": "int", "value": 1, "default": 1),
                "invoice_no": "string",
                "date": ("type": "date", "format": "YYYYMMDD"),
            ),
            "item": (
                "type": ("type": "int", "value": 2, "default": 2),
                "item_no": "string",
                "item": "string",
                "pcs": "int",
                "price": "number",
            ),
            "footer": (
                "type": ("type": "int", "value": 3, "default": 3),
                "total": "number",
            ),
        );
        hash opts = hash(
        );

        # Iterate through source
        AbstractCsvIterator i = new CsvDataIterator(CsvMultiInput, spec, ("extended_record": True) + opts);

        #list l = map $1, i;
        testAssertion("multi iterator ", \equalsIterated(), (new ListIterator(CsvMultiRecords), i));

        CsvStringWriter w( spec, ("write-headers": False, "optimal-quotes" : True, "quote_escape" : '"')+opts );
        string outstr = w.write(CsvMultiRecords);
        testAssertion("multi writer", \equals(), (CsvMultiInput, outstr));

        # remove type and test default value
        list CsvMultiRecords2 = ();
        foreach hash item in (CsvMultiRecords) {
            hash r = item;
            remove r.record.type;
            push CsvMultiRecords2, r;
        }
        w = new CsvStringWriter( spec, ("write-headers": False, "optimal-quotes" : True, "quote_escape" : '"')+opts );
        outstr = w.write(CsvMultiRecords2);
        testAssertion("multi writer default value", \equals(), (CsvMultiInput, outstr));

        # Iterate through source
        i = new CsvDataIterator(CsvMultiInput, spec, ("extended_record": False) + opts);

        CsvMultiRecords2 = map ($1.record), CsvMultiRecords;
        testAssertion("multi iterator old record", \equalsIterated(), (new ListIterator(CsvMultiRecords2), i));

        # Iterate through source
        i = new CsvIterator(new StringInputStream(CsvMultiInput), NOTHING, spec, ("extended_record": False) + opts);

        CsvMultiRecords2 = map ($1.record), CsvMultiRecords;
        testAssertion("multi iterator old record stream", \equalsIterated(), (new ListIterator(CsvMultiRecords2), i));
    }

    csvOldStyleMultiTest() {
        list CsvMultiRecords2 = (
            ("1","2016-01","20160401"),
            ("2","123","Sony, Xperia S","1","100"),
            ("2","124","Nokia, Lumia","2","150"),
            ("3","250"),
        );
        hash opts = hash(
        );

        # Iterate through source
        CsvDataIterator i(CsvMultiInput, ("header_names": False, "header_lines": 0,)+opts);
        list recs1 = ();
        list recs2 = ();
        while (i.next()) {
            push recs1, i.getValue();
            push recs2, i.getRecordList();
        }
        list CsvMultiRecords3 = ();
        foreach list item in (CsvMultiRecords2) {
            push CsvMultiRecords3, (map {$#: $1}, item);
        }
        testAssertionValue("Old style multi CsvDataIterator hash", recs1, CsvMultiRecords3);
        testAssertionValue("Old style multi CsvDataIterator list", recs2, CsvMultiRecords2);

        CsvStringWriter w( ("write-headers": False, "optimal-quotes" : True, "quote_escape" : '"')+opts );
        string outstr = w.write(CsvMultiRecords2);
        testAssertion("multi writer", \equals(), (CsvMultiInput, outstr));
    }

    csvNonNumeric() {
        hash opts = ("fields": ("int": "int", "float": "float", "number": "number"));

        CsvDataIterator i("1x2,1.2x,1e2n3", opts);
        i.next();
        testAssertion("Non-numeric field value", \i.getValue(), (), new TestResultExceptionType("FIELD-VALUE-ERROR"));
    }

    escapeTest() {
        list row = ( ( '12"', 'a"bc"', ), );
        string esc_default = '"12\"","a\"bc\""'+"\n";
        string esc_double = '"12""","a""bc"""'+"\n";

        CsvStringWriter w( ("optimal-quotes" : False) );
        string outstr = w.write(row);
        testAssertion("default escaping", \equals(), (esc_default, outstr));

        w = new CsvStringWriter(("optimal_quotes": False) );
        outstr = w.write(row);
        assertEq(esc_default, outstr);

        w = new CsvStringWriter( ("optimal-quotes" : False, "quote_escape" : '"') );
        outstr = w.write(row);
        testAssertion("double escaping", \equals(), (esc_double, outstr));

        hash dmh = ("cc": "XX");
        w = new CsvStringWriter({
            "headers": ("XX", "serno", "desc", "received", "activated"),
            "fields": {
                "received": {"type": "date", "format": "DDMMYYYY"},
                "activated": "bool",
            },
            "datamap": dmh,
        });
        outstr = w.write(CsvRecords.iterator());
        assertEq("XX,serno,desc,received,activated\n" + CsvInput, outstr);
    }

    csvData() {
        hash opts = ("fields": ("id": "int", "name": "string", "descr": "string"));
        string CsvInput = "1,UK,United Kingdom
2,AT,Austria
3,CZ,Czech rep.
4,IE,Ireland
";
        list Output = (
            {"id": 1, "name": "UK", "descr": "United Kingdom"},
            {"id": 2, "name": "AT", "descr": "Austria"},
            {"id": 3, "name": "CZ", "descr": "Czech rep."},
            {"id": 4, "name": "IE", "descr": "Ireland"},
        );

        # Iterate through source
        CsvDataIterator i(CsvInput, opts);

        testAssertion("CsvDataIterator 2", \equalsIterated(), (i, new ListIterator(Output)));
    }

    csvSingleTest() {
        hash spec = (
            "<single>": (
                "id": ("type": "int"),
                "name": "string",
                "group": ("type": "int", "default": 1),
                "image": ("type": "string", "header": "picture", "default": "empty.jpg"),
                "date": ("type": "*date", "header": "created", "format": "YYYYMMDD"),
                "ver": ("type": "int"),
            ),
        );
        hash opts = hash();

        list Records = (
            ("id": 1, "name": "Gogo", "group": 5, "date": 2016-01-31, "ver": 1),
            ("id": 2, "name": "Bobo", "image": "bobo.jpg", "ver": 3),
            ("id": 3, "name": "Huhu", "ver": 5),
        );
        string output = "id,name,group,picture,created,ver
1,Gogo,5,empty.jpg,20160131,1
2,Bobo,1,bobo.jpg,,3
3,Huhu,1,empty.jpg,,5
";
        CsvStringWriter w(spec, ("write_headers": True, "optimal-quotes" : True, "quote_escape" : '"')+opts);
        string outstr = w.write(Records);
        testAssertionValue("CsvDataWriter test default and header", outstr, output);

        CsvDataIterator i(outstr, spec, ("header_names": True, "header_lines": 1,)+opts);
        list rec2 = Records;
        # add default values
        rec2[0].image = spec{CSV_TYPE_SINGLE}.image.default;
        rec2[1].group = spec{CSV_TYPE_SINGLE}.group.default;
        rec2[1].date = NOTHING;
        rec2[2].image = spec{CSV_TYPE_SINGLE}.image.default;
        rec2[2].group = spec{CSV_TYPE_SINGLE}.group.default;
        rec2[2].date = NOTHING;

        testAssertion("single iterator header", \equalsIterated(), (new ListIterator(rec2), i));
    }

    csvSingleMapTest() {
        hash spec = (
            CSV_TYPE_SINGLE: (
                "col1": ("type": "int", "index": 4),
                "col2": ("type": "int", "index": 0),
                "col3": ("type": "int", "index": 3, "header": "COL3"),
                "col4": ("type": "int", "index": 2),
                "col5": ("type": "int", "index": 1),
            ),
        );
        hash opts = hash();

        list Records = (
            ("col1": 1, "col2": 2, "col3": 3, "col4": 4, "col5": 5),
            (10, 20, 30, 40, 50),
        );
        string output = "col2,col5,col4,COL3,col1
2,5,4,3,1
20,50,40,30,10
";
        CsvStringWriter w(spec, ("write_headers": True, "optimal-quotes" : True, "quote_escape" : '"')+opts);
        string outstr = w.write(Records);
        testAssertionValue("CsvDataWriter test map", outstr, output);

        # remove idx
        foreach string k in (spec{CSV_TYPE_SINGLE}.keyIterator()) {
            remove spec{CSV_TYPE_SINGLE}{k}.index;
        }

        # issue #2179 added a negative test
        {
            hash nspec = spec;
            nspec{CSV_TYPE_SINGLE}.xxx = "int";
            CsvDataIterator i(outstr, nspec, ("header_names": True, "header_lines": 1, "header_reorder": False)+opts);
            assertThrows("ABSTRACTCSVITERATOR-ERROR", \i.next());
        }

        AbstractCsvIterator i = new CsvDataIterator(outstr, spec, ("header_names": True, "header_lines": 1,)+opts);
        hash r2 = hash();
        # add keys
        foreach int item in (Records[1]) {
            r2{sprintf("col%d", $#+1)} = item;
        }
        list Recsh2 = ();
        push Recsh2, Records[0];
        push Recsh2, r2;
        list Recsl2;
        list lines = output.split("\n");

        push Recsl2, map ($1.toInt()), lines[1].split(",");
        push Recsl2, map ($1.toInt()), lines[2].split(",");

        list rh;
        list rl;
        while (i.next()) {
            push rh, i.getValue();
            push rl, i.getRecordList();
        }
        testAssertionValue("single iterator remap header test hash", rh, Recsh2);
        testAssertionValue("single iterator remap header test list", rl, Recsl2);

        hash spec2 = spec;
        remove spec2{CSV_TYPE_SINGLE}{"col1"};
        remove spec2{CSV_TYPE_SINGLE}{"col4"};

        w = new CsvStringWriter(("fields": spec2{CSV_TYPE_SINGLE}, "headers": ("col2","col5","col4","COL3","col1")));
        outstr = w.write(Recsl2);
        testAssertionValue("CsvDataWriter test map partial reorder", outstr, output);

        w = new CsvStringWriter(("fields": spec2{CSV_TYPE_SINGLE}, "headers": ("col2","col5","col4","COL3","col1"), "header_reorder": False));
        outstr = w.write(((2,3,5,4,1),(20,30,50,40,10)));
        testAssertionValue("CsvDataWriter test map partial noreorder", outstr, output);

        StringOutputStream outstream();
        CsvWriter writer(outstream, ("fields": spec2{CSV_TYPE_SINGLE}, "headers": ("col2","col5","col4","COL3","col1"), "header_reorder": False));
        writer.write(((2,3,5,4,1),(20,30,50,40,10)));
        assertEq(output, outstream.getData());

        i = new CsvDataIterator(outstr, spec2, ("header_names": True, "header_lines": 1)+opts);
        rh = ();
        rl = ();
        while (i.next()) {
            push rh, i.getValue();
            push rl, i.getRecordList();
        }
        list Recsh2str = Recsh2;
        Recsh2str[0].col1 = string(Recsh2str[0].col1);
        Recsh2str[0].col4 = string(Recsh2str[0].col4);
        Recsh2str[1].col1 = string(Recsh2str[1].col1);
        Recsh2str[1].col4 = string(Recsh2str[1].col4);
        list Recsl2str = Recsl2;
        Recsl2str[0][2] = string(Recsl2str[0][2]);
        Recsl2str[0][4] = string(Recsl2str[0][4]);
        Recsl2str[1][2] = string(Recsl2str[1][2]);
        Recsl2str[1][4] = string(Recsl2str[1][4]);

        testAssertionValue("single iterator remap header test hash partial spec reorder", rh, Recsh2str);
        testAssertionValue("single iterator remap header test list partial spec reorder", rl, Recsl2str);

        i = new CsvDataIterator(outstr, spec, ("header_names": True, "header_lines": 1, "header_reorder": False)+opts);
        rh = ();
        rl = ();
        while (i.next()) {
            push rh, i.getValue();
            push rl, i.getRecordList();
        }
        testAssertionValue("single iterator remap header test hash noreorder", rh, Recsh2);
        Recsl2 = ();
        push Recsl2, map ($1.value), Records[0].pairIterator();
        push Recsl2, Records[1];
        testAssertionValue("single iterator remap header test list noreorder", rl, Recsl2);

        i = new CsvDataIterator(outstr, spec2, ("header_names": True, "header_lines": 1, "header_reorder": False)+opts);
        rh = ();
        rl = ();
        while (i.next()) {
            push rh, i.getValue();
            push rl, i.getRecordList();
        }
        testAssertionValue("single iterator remap header test hash partial spec noreorder", rh, Recsh2str);
        Recsl2str = ();
        push Recsl2str, (Records[0].col2, Records[0].col3, Records[0].col5, string(Records[0].col4), string(Records[0].col1));
        push Recsl2str, (Records[1][1], Records[1][2], Records[1][4], string(Records[1][3]), string(Records[1][0]));
        testAssertionValue("single iterator remap header test list partial spec noreorder", rl, Recsl2str);

        i = new CsvIterator(new StringInputStream(outstr), outstr.encoding(), spec2, ("header_names": True, "header_lines": 1, "header_reorder": False)+opts);
        rh = ();
        rl = ();
        while (i.next()) {
            push rh, i.getValue();
            push rl, i.getRecordList();
        }
        testAssertionValue("single iterator remap header test hash partial spec noreorder", rh, Recsh2str);
        Recsl2str = ();
        push Recsl2str, (Records[0].col2, Records[0].col3, Records[0].col5, string(Records[0].col4), string(Records[0].col1));
        push Recsl2str, (Records[1][1], Records[1][2], Records[1][4], string(Records[1][3]), string(Records[1][0]));
        testAssertionValue("single iterator remap header test list partial spec noreorder", rl, Recsl2str);
    }

    csvConfig() {
        assertThrows("ABSTRACTCSVITERATOR-ERROR",
            CsvDataIterator sub() {return new CsvDataIterator(
                "1,2,3,4,5",
                (
                    CSV_TYPE_SINGLE: (
                        "col1": ("type": "int", "index": 0),
                        "col2": ("type": "int", "index": 0),
                        "col3": ("type": "int", "index": 2),
                        "col4": ("type": "int", "index": 3),
                        "col5": ("type": "int", "index": 4),
                    ),
                ),
                {}
            );}, (), NOTHING);
    }

    csvMultiResolveTest() {
        string Data =
            "00000,ABCDEABCDE\n"+
            "00001,12345\n"+
            "90000,A\n"+
            "90000,QQQABCDEAB\n"+
            "91230,12345\n"+
            "12345\n"+
            "12,KID,01122016\n"+
            "45,CHYCHOTA_SE,1\n"+
            "ABCD,A,AA,AAA\n"+
            "1234,A,1,2\n";

        hash Specs = (
            "type2_value1_0" : (
                "col1" : (
                    "type"   : "int",
                    "value"  : "00000",
                ),
                "col2" : (
                    "type"   : "string",
                ),
            ),
            "type2_value1_1" : (
                "col1" : (
                    "type"   : "int",
                    "value"  : 1,
                ),
                "col2" : (
                    "type"   : "int",
                ),
            ),
            "type2_value1_90000_regex2" : (
                "col1" : (
                    "type"   : "int",
                    "value"  : 90000,  # earlier than regex matching value too
                ),
                "col2" : (
                    "type"   : "string",
                    "regex"  : "^QQQ", # 2nd rule
                ),
            ),
            "type2_value1_90000" : (
                "col1" : (
                    "type"   : "int",
                    "value"  : 90000,  # earlier than regex matching value too
                ),
                "col2" : (
                    "type"   : "string",
                ),
            ),
            "type2_regex1_9xxx0" : (
                "col1" : (
                    "type"   : "int",
                    "regex"  : "^9[0-9]{3,3}0",
                ),
                "col2" : (
                    "type"   : "int",
                ),
            ),
            "type1_norule" : (
                "col1" : (
                    "type"   : "int",
                ),
            ),
            "type3_value2_KID" : (
                "col1" : (
                    "type"   : "int",
                ),
                "col2" : (
                    "type"   : "string",
                    "value"  : "KID",
                ),
                "col3" : (
                    "type"     : "date",
                    # "format"   : "DDMMYYYY",
                    # "timezone" : "UTC", # use global if omitted
                ),
            ),
            "type3_regex2_ch" : (
                "col1" : (
                    "type"   : "int",
                ),
                "col2" : (
                    "type"   : "string",
                    "regex"  : "^CH",
                ),
                "col3" : (
                    "type"   : "int",
                ),
            ),
            "type4_1" : (
                "col1" : (
                    "type"   : "string",
                ),
                "col2" : (
                    "type"   : "string",
                ),
                "col3" : (
                    "type"   : "string",
                ),
                "col4" : (
                    "type"   : "string",
                ),
            ),
            "type4_2" : (
                "col1" : (
                    "type"   : "int",
                ),
                "col2" : (
                    "type"   : "string",
                ),
                "col3" : (
                    "type"   : "int",
                ),
                "col4" : (
                    "type"   : "int",
                ),
            ),
        );

        hash GlobalOptions = (
            "encoding"     : "UTF-8",
            "eol"          : "\n",
            "ignore_empty" : True,
            "timezone"     : "UTC", # used if not in some date column specification
            "extended_record": True,
            "date_format"  : "DDMMYYYY",
            );

        list Result = (
            ("type": "type2_value1_0", "record": ("col1": 0, "col2": "ABCDEABCDE")),
            ("type": "type2_value1_1", "record": ("col1": 1, "col2": 12345)),
            ("type": "type2_value1_90000", "record": ("col1": 90000, "col2": "A")),
            ("type": "type2_value1_90000_regex2", "record": ("col1": 90000, "col2": "QQQABCDEAB")),
            ("type": "type2_regex1_9xxx0", "record": ("col1": 91230, "col2": 12345)),

            ("type": "type1_norule", "record": ("col1": 12345)),

            ("type": "type3_value2_KID", "record": ("col1": 12, "col2": "KID", "col3": 2016-12-01Z)),
            ("type": "type3_regex2_ch", "record": ("col1": 45, "col2": "CHYCHOTA_SE", "col3": 1)),

            ("type": "type4_1", "record": ("col1": "ABCD", "col2": "A", "col3": "AA", "col4": "AAA")),
            ("type": "type4_2", "record": ("col1": 1234, "col2": "A", "col3": 1, "col4": 2)),
        );
        MyCsvDataIterator i(Data, Specs, GlobalOptions);
        testAssertion("multi type resolving", \equalsIterated(), (new ListIterator(Result), i));
    }

    csvFileIterator() {
        string file = tmp_location() + "/test-file.csv";
        unlink(file);
        {
            File f();
            f.open2(file, O_WRONLY | O_CREAT);
            f.write("a,b,c\n1,2,3\n");
        }
        on_exit unlink(file);

        CsvFileIterator i(file);
        assertEq(True, i.hasCallableMethod("hstat"));
        assertEq(True, i.hasCallableMethod("stat"));
        assertEq(True, i.hasCallableMethod("getFileName"));
        assertEq(True, i.hasCallableMethod("getEncoding"));

        assertEq((("0": "a", "1": "b", "2": "c"), ("0": "1", "1": "2", "2": "3")), (map $1, i));
    }

    issue2739() {
        string file = tmp_location() + "/test-issue2739.csv";
        on_exit unlink(file);

        File f();
        f.open2(file, O_WRONLY | O_CREAT);
        f.write(
            "abcd,1234,2072-02-29\n"
            "\"efgh\",\"5678\",\"2096-02-29\"\n"
        );

        CsvFileIterator i(file, {
            'fields': {'S': 'string',
                       'N': 'number',
                       'D': {'type': 'date', 'format': 'YYYY-MM-DD'}}
        });

        i.next();
        assertEq("abcd,1234,2072-02-29", i.getRawLine());
        assertEq(("abcd", "1234", "2072-02-29"), i.getRawLineValues());
        assertNeq(("abcd", 1234, "2072-02-29"), i.getRawLineValues());
        assertNeq(("abcd", "1234", date("2072-02-29")), i.getRawLineValues());
        assertEqSoft({"S" : "abcd", "N": 1234, "D": date("2072-02-29")}, i.getValue());

        i.next();
        assertEq("\"efgh\",\"5678\",\"2096-02-29\"", i.getRawLine());
        assertEq(("efgh", "5678", "2096-02-29"), i.getRawLineValues());
        assertNeq(("efgh", 5678, "2096-02-29"), i.getRawLineValues());
        assertNeq(("efgh", "5678", date("2096-02-29")), i.getRawLineValues());
        assertEqSoft({"S" : "efgh", "N": 5678, "D": date("2096-02-29")}, i.getValue());
    }
}

class MyCsvDataIterator inherits CsvDataIterator {
    constructor(string data, hash spec, *hash opts) : CsvUtil::CsvDataIterator(data, spec, opts) {
    }
    string identifyType(list values) {
        if (values.size() != 4) {
            return CsvDataIterator::identifyType(values);
        }
        switch (values[0]) {
        case /^[0-9]/:
            return "type4_2";
        default:
            return "type4_1";
        }
    }
}
