#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

%new-style
%enable-all-warnings
%require-types
%strict-args

%requires ../../../../qlib/QUnit.qm
%requires ../../../../qlib/DataProvider
%requires ../../../../qlib/FixedLengthUtil

%exec-class Test

class Test inherits QUnit::Test {
    public {
        const StrData = "11111bb  \ncddd31122014\n22222gggg\n";
        const StrData2 = "00001aaaa\n00002bbbb\n00001cccc\n99999dddd\n";

        const Specs = (
            "type1" : (
                "col1" : (
                    "length" : 5,
                    "type"   : "int",
                ),
                "col2" : (
                    "length" : 4,
                    "type"   : "string",
                ),
            ),
            "type2" : (
                "col3" : (
                    "length" : 1,
                    "type"   : "string",
                ),
                "col4" : (
                    "length" : 3,
                    "type"   : "string",
                ),
                "col5" : (
                    "length"   : 8,
                    "type"     : "date",
                    "format"   : "DDMMYYYY",
                    # "timezone" : "UTC", # use global if omitted
                ),
            ),
        );

        const AmbiguousData = "AABC\nBXYZ\nA123\nB789\n";

        const AmbiguousSpecs = (
            "type1" : (
                "col1" : (
                    "length": 1,
                    "value": "A"
                ),
                "col2": (
                    "length": 3,
                ),
            ),
            "type2" : (
                "col1" : (
                    "length": 1,
                    "value": "B"
                ),
                "col2": (
                    "length": 3,
                ),
            ),
        );

        # Incorrect use
        const RedundantSpecs = (
            "type1" : (
                "col1" : (
                    "length" : 5,
                    "type"   : "int",
                ),
                "col2" : (
                    "length" : 4,
                    "type"   : "string",
                ),
            ),
            "type2" : (
                "col1" : (
                    "length" : 5,
                    "type"   : "int",
                ),
                "col2" : (
                    "length" : 4,
                    "type"   : "string",
                ),
            ),
        );

        # test default value
        const SpecsWithDefault = (
            "type1" : (
                "col1" : (
                    "length" : 5,
                    "type"   : "int",
                    "default"  : "00001",
                ),
                "col2" : (
                    "length" : 4,
                    "type"   : "string",
                ),
            ),
            "type2" : (
                "col1" : (
                    "length" : 5,
                    "type"   : "int",
                    "default"  : "00002",
                ),
                "col2" : (
                    "length" : 4,
                    "type"   : "string",
                ),
            ),
        );

        const HashData = (
            ("type": "type1", "record": ("col1": 11111, "col2": "bb")),
            ("type": "type2", "record": ("col3": "c", "col4": "ddd", "col5": 2014-12-31Z)),
            ("type": "type1", "record": ("col1": 22222, "col2": "gggg")),
        );

        const HashData2 = (
            ("type": "type1", "record": ("col2": "aaaa")),
            ("type": "type2", "record": ("col2": "bbbb")),
            ("type": "type1", "record": ("col2": "cccc")),
            ("type": "type1", "record": ("col1": "99999", "col2": "dddd")),
        );

        const HashDataTS = (("type": "type1", "record": ("col1": 11111, "col2": "b\tb")),);

        const GlobalOptions = (
            "encoding"     : "UTF-8",
            "eol"          : "\n",
            "ignore_empty" : True,
            "timezone"     : "UTC", # used if not in some date column specification
        );

        # Writer
        const GlobalWriterOptions = (
            "eol" : "\n",
        );

        # padding tests
        const PADDING_SPEC = (
            "r1": (
                "c1": ( "length": 5, "type": "string", "padding_char": "X"),
                "c2": ( "length": 5, "type": "string", "padding": "left", "padding_char": "0"),
                "c3": ( "length": 5, "type": "string", "padding": "right", "padding_char": "0"),
                "c4": ( "length": 5, "type": "string", "padding": "left", "padding_char": " "),
            ),
            "r2": (
                "c1": ( "length": 5, "type": "int"),
                "c2": ( "length": 5, "type": "int", "padding_char": "0"),
                "c3": ( "length": 5, "type": "int", "padding_char": "0"),
            ),
            "r3": (
                "c1": ( "length": 6, "type": "float"),
                "c2": ( "length": 6, "type": "float", "padding_char": "0"),
                "c3": ( "length": 6, "type": "float", "padding_char": "0"),
                "c4": ( "length": 8, "type": "float", "padding_char": "0"),
            ),
        );

        const PADDING_DATA1 = (
            ("type": "r1", "record": ("c1": "a", "c2": "a", "c3": "a", "c4": "a")));
        const PADDING_DATA2 = (
            ("type": "r2", "record": ("c1": -1, "c2": +1, "c3": -1)));
        const PADDING_DATA3 = (
            ("type": "r3", "record": ("c1": -1.2, "c2": 1.2, "c3": -1.2, "c4": -2e-34)));

        const PADDING_OUTPUT1 = "aXXXX0000aa0000    a\n";
        const PADDING_OUTPUT2 = "   -100001-0001\n";
        const PADDING_OUTPUT3 = "  -1.20001.2-001.2-002e-34\n";
    }

    constructor() : QUnit::Test("FixedLengthUtil", "1.0", \ARGV) {
        addTestCase("data provider", \dataProviderTest());
        addTestCase("FixedLengthUtil basic tests", \basicTests());
        addTestCase("FixedLengthUtil padding test #1", \test_padding_1());
        addTestCase("FixedLengthUtil padding test #2", \test_padding_2());
        addTestCase("FixedLengthUtil padding test #3", \test_padding_3());
        addTestCase("FixedLengthUtil value-too-large test", \test_value_too_large());
        addTestCase("FixedLengthUtil truncate tests", \test_truncate());
        addTestCase("FixedLengthUtil tabs-to-spaces reader tests", \test_reading_data_with_tabs());
        addTestCase("FixedLengthUtil tabs-to-spaces writer tests", \test_writing_data_with_tabs());
        set_return_value(main());
    }

    dataProviderTest() {
        list<hash<auto>> rows;
        list<hash<auto>> rows2;
        {
            FixedLengthReadDataProvider dp;

            {
                dp = DataProvider::getFactoryObjectWithTemplate("fixedlengthread", {
                    "path": "$template:value",
                }, {
                    "spec": Specs,
                } + GlobalOptions);
                hash<DataProviderInfo> info = dp.getInfo();
                assertEq(Type::String, info.name.type());
                assertTrue(info.has_record);
                assertTrue(info.supports_read);
                assertEq(Type::String, info.desc.type());
            }

            # the fixed-length data can only be iterated once and then the provide needs to be recreated
            code reset = sub (string strdata = StrData, hash<auto> specs = Specs) {
                dp = new FixedLengthReadDataProvider(new StringInputStream(strdata), specs, GlobalOptions);
            };

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertFalse(info.supports_bulk_read);
            }

            {
                reset();
                hash<DataProviderExpression> cond = <DataProviderExpression>{
                    "exp": DP_OP_OR,
                    "args": (
                        <DataProviderExpression>{
                            "exp": DP_SEARCH_OP_EQ,
                            "args": (
                                <DataProviderFieldReference>{"field": "record.col1"},
                                11111,
                            ),
                        },
                        <DataProviderExpression>{
                            "exp": DP_SEARCH_OP_EQ,
                            "args": (
                                <DataProviderFieldReference>{"field": "record.col3"},
                                "c",
                            ),
                        },
                    )
                };
                rows = map $1, dp.searchRecords(cond);
                assertEq(2, rows[0].size());
                assertEq(2, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords();
                assertEq(2, rows[0].size());
                assertEq(2, rows[0].record.size());
                assertEq(3, rows.size());

                reset();
                *hash<auto> row = dp.searchSingleRecord({"type": "type1", "record.col1": 11111});
                assertEq(Type::Hash, row.type());
                assertEq("bb", row.record.col2);
                assertEq(Type::String, row.record.col2.type());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({}, {"columns": "col1"});
                assertEq(2, rows[0].size());
                assertEq(1, rows[0].record.size());
                assertEq(3, rows.size());
            }

            {
                reset();
                rows = map $1, dp.searchRecords({}, {"limit": 1, "offset": 1});
                assertEq(2, rows[0].size());
                assertEq(3, rows[0].record.size());
                assertEq(1, rows.size());
                hash<auto> row1 = rows[0];

                reset();
                rows = map $1, dp.searchRecords({}, {"limit": 1, "offset": 2});
                assertEq(2, rows[0].size());
                assertEq(2, rows[0].record.size());
                assertEq(1, rows.size());
                hash<auto> row2 = rows[0];

                reset();
                rows = map $1, dp.searchRecords();
                assertEq(2, rows[0].size());
                assertEq(2, rows[0].record.size());
                assertEq(3, rows.size());

                assertEq(rows[1], row1);
                assertEq(rows[2], row2);
            }

            {
                reset();
                AbstractDataProviderBulkRecordInterface i = dp.getBulkRecordInterface();
                hash<auto> q = i.getValue();
                assertEq(Type::List, q.firstValue().type());
                assertEq(3, q.firstValue().lsize());
            }

            {
                reset(AmbiguousData, AmbiguousSpecs);
                rows2 = map $1, dp.searchRecords();
                assertEq(4, rows2.size());

                reset(AmbiguousData, AmbiguousSpecs);
                *hash<auto> row = dp.searchSingleRecord({"type": "type2", "record.col2": "XYZ"});
                assertEq(Type::Hash, row.type());
                assertEq("B", row.record.col1);
                assertEq(Type::String, row.record.col2.type());
            }

        }

        {
            FixedLengthReadDataProvider dp;

            # the fixed-length data can only be iterated once and then the provide needs to be recreated
            code reset = sub (string strdata = StrData, hash<auto> specs = Specs) {
                dp = new FixedLengthReadDataProvider({
                    "stream": new StringInputStream(strdata),
                    "spec": specs,
                 } + GlobalOptions);
            };

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertFalse(info.supports_bulk_read);
            }

            {
                reset();
                rows = map $1, dp.searchRecords();
                assertEq(3, rows.size());

                reset();
                *hash<auto> row = dp.searchSingleRecord({"type": "type1", "record.col1": 11111});
                assertEq(Type::Hash, row.type());
                assertEq("bb", row.record.col2);
                assertEq(Type::String, row.record.col2.type());
            }

            {
                reset();
                AbstractDataProviderBulkRecordInterface i = dp.getBulkRecordInterface();
                hash<auto> q = i.getValue();
                assertEq(Type::List, q.firstValue().type());
                assertEq(3, q.firstValue().lsize());
            }

            {
                reset(AmbiguousData, AmbiguousSpecs);
                rows2 = map $1, dp.searchRecords();
                assertEq(4, rows2.size());

                reset(AmbiguousData, AmbiguousSpecs);
                *hash<auto> row = dp.searchSingleRecord({"type": "type2", "record.col2": "XYZ"});
                assertEq(Type::Hash, row.type());
                assertEq("B", row.record.col1);
                assertEq(Type::String, row.record.col2.type());
            }

        }

        {
            FixedLengthWriteDataProvider dp;
            StringOutputStream stream;

            # the fixed-length data can only be iterated once and then the provide needs to be recreated
            code reset = sub (hash<auto> specs = Specs) {
                stream = new StringOutputStream();
                dp = new FixedLengthWriteDataProvider(stream, specs, GlobalWriterOptions);
            };

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertTrue(info.supports_create);
                assertFalse(info.supports_bulk_read);

                map dp.createRecord($1), rows;

                assertEq(StrData, stream.getData());
            }

            {
                reset(AmbiguousSpecs);
                map dp.createRecord($1), rows2;
                assertEq(AmbiguousData, stream.getData());
            }
        }

        {
            FixedLengthWriteDataProvider dp;
            StringOutputStream stream;

            # the fixed-length data can only be iterated once and then the provide needs to be recreated
            code reset = sub (hash<auto> specs = Specs) {
                stream = new StringOutputStream();
                dp = new FixedLengthWriteDataProvider({
                    "stream": stream,
                    "spec": specs,
                } +GlobalWriterOptions);
            };

            {
                reset();
                hash<DataProviderInfo> info = dp.getInfo();
                assertTrue(info.has_record);
                assertTrue(info.supports_create);
                assertFalse(info.supports_bulk_read);

                map dp.createRecord($1), rows;

                assertEq(StrData, stream.getData());
            }

            {
                reset(AmbiguousSpecs);
                map dp.createRecord($1), rows2;
                assertEq(AmbiguousData, stream.getData());
            }
        }
    }

    basicTests() {
        FixedLengthAbstractIterator i = new FixedLengthDataIterator(StrData, Specs, GlobalOptions);
        testAssertionValue("line 1 present", i.next(), True);
        testAssertionValue("line 1 content check", i.getValue(), HashData[0]);
        testAssertionValue("line 2 present", i.next(), True);
        testAssertionValue("line 2 content check", i.getValue(), HashData[1]);
        testAssertionValue("line 3 present", i.next(), True);
        testAssertionValue("line 3 content check", i.getValue(), HashData[2]);
        testAssertionValue("line 4 is not there", i.next(), False);

        i = new FixedLengthDataIterator(StrData, RedundantSpecs, GlobalOptions);

        code test_closure = sub () {
            i.next();
            i.getValue();
        };
        testAssertion("Line with byte length", test_closure, NOTHING, new TestResultExceptionType("FIXED-LENGTH-UTIL-NON-MATCHING-TYPE"));

        FixedLengthDataWriter w(Specs, GlobalWriterOptions);
        testAssertionValue("file content check", w.write(HashData), StrData);

        w = new FixedLengthDataWriter(SpecsWithDefault, GlobalWriterOptions);
        testAssertionValue("file content check", w.write(HashData2), StrData2);

        StringOutputStream output();
        FixedLengthWriter writer(output, Specs, GlobalWriterOptions);
        writer.write(HashData);
        assertEq(StrData, output.getData());

        writer = new FixedLengthWriter(output, SpecsWithDefault, GlobalWriterOptions);
        writer.write(HashData2);
        assertEq(StrData2, output.getData());
    }

    test_padding_1() {
        FixedLengthDataWriter pw1(PADDING_SPEC);
        testAssertionValue("file content check - padding of strings", pw1.write(PADDING_DATA1), PADDING_OUTPUT1);
    }

    test_padding_2() {
        FixedLengthDataWriter pw2(PADDING_SPEC);
        testAssertionValue("file content check - padding of integers", pw2.write(PADDING_DATA2), PADDING_OUTPUT2);
    }

    test_padding_3() {
        FixedLengthDataWriter pw3(PADDING_SPEC);
        testAssertionValue("file content check - padding of floats", pw3.write(PADDING_DATA3), PADDING_OUTPUT3);
    }

    test_value_too_large() {
        list data1 = HashData;
        data1[2].record.col2 += data1[2].record.col2;
        FixedLengthDataWriter w (Specs, GlobalWriterOptions);
        assertThrows ("FIELD-INPUT-ERROR", NOTHING, sub () { w.write(data1); }, (), "throw value too large");
    }

    test_truncate() {
        list data1 = HashData;
        data1[2].record.col2 += data1[2].record.col2;
        {
            hash specs = Specs;
            specs.type1.col2.truncate = True;
            FixedLengthDataWriter w (specs, GlobalWriterOptions);
            assertEq (StrData, w.write(data1), "truncate - field option");
        }
        {
            hash gopts = GlobalWriterOptions + ("truncate": True);
            FixedLengthDataWriter w (Specs, gopts);
            assertEq (StrData, w.write(data1), "truncate - global option");
        }
    }

    test_reading_data_with_tabs() {
        string str = "11111b\tb \n";
        {
            FixedLengthDataIterator i (str, Specs, GlobalOptions);
            assertTrue(i.next(), "tabs to spaces - reader default case - line present");
            assertEq(HashDataTS[0], i.getValue(), "tabs to spaces - reader default case - line content check");
        }
        for (int i=0; i<10; ++i) {
            hash specs = Specs;
            specs.type1.col2.tab2space = i;
            specs.type1.col2.truncate = True;
            list data1 = HashDataTS;
            data1[0].record.col2 = substr ("b" + strmul(' ', i) + "b", 0, specs.type1.col2.length);
            FixedLengthDataIterator r(str, specs, GlobalOptions);
            assertTrue(r.next(), sprintf("tabs to spaces - reader field option - case #%d - line present", i));
            assertEq(data1[0], r.getValue(), sprintf("tabs to spaces - reader field option - case #%d - line content check", i));
        }
        for (int i=0; i<10; ++i) {
            hash gopts = GlobalOptions + ("tab2space": i, "truncate": True);
            list data1 = HashDataTS;
            data1[0].record.col2 = substr("b" + strmul(' ', i) + "b", 0, Specs.type1.col2.length);
            FixedLengthDataIterator r(str, Specs, gopts);
            assertTrue(r.next(), sprintf("tabs to spaces - reader global option - case #%d - line present", i));
            assertEq(data1[0], r.getValue(), sprintf("tabs to spaces - reader global option - case #%d - line content check", i));
        }
    }

    test_writing_data_with_tabs() {
        {
            string str = "11111b\tb \n";
            FixedLengthDataWriter w(Specs, GlobalWriterOptions);
            assertEq(str, w.write (HashDataTS), "tabs to spaces - writer default case");
        }
        for (int i=0; i<10; ++i) {
            hash specs = Specs;
            specs.type1.col2.tab2space = i;
            specs.type1.col2.truncate = True;
            string str = "11111" + substr("b" + strmul(' ', i) + "b  ", 0, Specs.type1.col2.length) + "\n";
            FixedLengthDataWriter w(specs, GlobalWriterOptions);
            assertEq(str, w.write(HashDataTS), sprintf("tabs to spaces - writer field option - case #%d", i));
        }
        for (int i=0; i<10; ++i) {
            hash gopts = GlobalWriterOptions + ("tab2space": i, "truncate": True);
            string str = "11111" + substr("b" + strmul(' ', i) + "b  ", 0, Specs.type1.col2.length) + "\n";
            FixedLengthDataWriter w(Specs, gopts);
            assertEq(str, w.write(HashDataTS), sprintf("tabs to spaces - writer global option - case #%d", i));
        }
    }
}
