#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

%require-types
%enable-all-warnings
%new-style
%strict-args
%allow-injection

%requires ../../../../qlib/QUnit.qm
%requires ../../../../qlib/Logger.qm
%requires ../../../../qlib/DataProvider

%requires reflection

%exec-class DataProviderTest

class TestRecordProvider inherits AbstractDataProviderRecordIterator {
    private {
        bool ok = False;
    }

    bool next() {
        return ok = !ok;
    }

    bool valid() {
        return ok;
    }

    hash<auto> getValue() {
        if (!ok) {
            throw "INVALID-ITERATOR", "error";
        }

        return {
            "a": 1,
            "b": 2,
        };
    }
}

class InfiniteRecordProvider inherits AbstractDataProviderRecordIterator {
    bool next() {
        return True;
    }

    bool valid() {
        return True;
    }

    hash<auto> getValue() {
        return {"a": 1, "b": 2};
    }
}

class TestRecordProcessor inherits AbstractDataProcessor {
    public {
        string pfx;
        list<hash<auto>> recs;
        *hash<auto> crec;
        *bool raise_error;
    }

    constructor(string pfx, *hash<auto> crec, *bool raise_error) {
        self.pfx = pfx;
        self.crec = crec;
        self.raise_error = raise_error;
    }

    clear() {
        remove recs;
    }

    setRaiseError() {
        raise_error = True;
    }

    private submitImpl(code enqueue, auto rec) {
        if (raise_error) {
            remove raise_error;
            throw "ERROR", "error";
        }
        hash<auto> new_rec = (map {pfx + "-" + $1.key: $1.value}, rec.pairIterator()) + crec;
        recs += new_rec;
        enqueue(new_rec);
    }

    private bool supportsBulkApiImpl() {
        return True;
    }
}

class TestFlushRecordProcessor inherits TestRecordProcessor {
    private {
        *hash<auto> new_rec;
    }

    constructor(string pfx, *hash<auto> crec, *bool raise_error, *hash<auto> new_rec)
        : TestRecordProcessor(pfx, crec, raise_error) {
        self.new_rec = new_rec;
    }

    private flushImpl(code enqueue) {
        if (new_rec) {
            enqueue(new_rec);
        }
    }
}

class DuplicateRecordProcessor inherits AbstractDataProcessor {
    public {
        string pfx1;
        string pfx2;
    }

    constructor(string pfx1, string pfx2) {
        self.pfx1 = pfx1;
        self.pfx2 = pfx2;
    }

    private submitImpl(code enqueue, auto rec) {
        hash<auto> rec1 = (map {pfx1 + "-" + $1.key: $1.value}, rec.pairIterator());
        enqueue(rec1);
        hash<auto> rec2 = (map {pfx2 + "-" + $1.key: $1.value}, rec.pairIterator());
        enqueue(rec2);
    }

    private bool supportsBulkApiImpl() {
        return True;
    }
}

class RecordProcessorSink inherits AbstractDataProcessor {
    public {
        list<auto> recs;
    }

    private submitImpl(code enqueue, auto rec) {
        push recs, rec;
        enqueue(rec);
    }

    private bool supportsBulkApiImpl() {
        return True;
    }
}

class DummyRecordProcessor inherits AbstractDataProcessor {
    private submitImpl(code enqueue, auto rec) {
        enqueue({"x": 1});
    }

    private bool supportsBulkApiImpl() {
        return True;
    }
}

class TestBulkRecordIterator inherits AbstractDataProviderRecordIterator {
    bool next() {
        return False;
    }

    bool valid() {
        return False;
    }

    hash<auto> getValue() {
        throw "INVALID-ITERATOR", "error";
    }

    bool supportsBulkApi() {
        return True;
    }

    *AbstractDataProviderBulkRecordInterface getBulkApi() {
        return new TestBulkRecordProvider();
    }
}

class TestBulkRecordProvider inherits AbstractDataProviderBulkRecordInterface {
    private {
        bool ok = True;
    }

    constructor() : AbstractDataProviderBulkRecordInterface(2) {
    }

    hash<string, auto> getValueImpl() {
        if (ok) {
            ok = False;
            return {
                "a": (1, 2),
                "b": ("one", "two"),
            };
        }
        return {};
    }
}

class TestIntProcessor inherits AbstractDataProcessor {
    private {
        int inc;
        int sum = 0;
        *bool skip_odd;
    }

    constructor(int inc, *bool skip_odd) {
        self.inc = inc;
        self.skip_odd = skip_odd;
    }

    int getSum() {
        return sum;
    }

    reset() {
        sum = 0;
    }

    private submitImpl(code enqueue, auto rec) {
        if (skip_odd && rec > inc && (rec % 2)) {
            return;
        }
        rec += inc;
        sum += rec;
        enqueue(rec);
    }

    private bool supportsBulkApiImpl() {
        return False;
    }
}

class StringTestType inherits Type {
    #! Creates the object
    constructor() : Type("softstring") {
    }

    #! Returns the type name
    string getName() {
        return "teststring";
    }
}

public class DataProviderTest inherits QUnit::Test {
    constructor() : Test("DataProvider Test", "1.0") {
        addTestCase("string test", \stringTest());
        addTestCase("data test", \dataTest());
        addTestCase("compat test", \compatTest());
        addTestCase("pipeline test", \pipelineTest());
        addTestCase("type cache test", \typeCacheTest());
        addTestCase("test", \dataProviderTest());

        # Return for compatibility with test harness that checks return value.
        set_return_value(main());
    }

    stringTest() {
        AbstractDataProviderType t = AbstractDataProviderType::get(new StringTestType());
        assertEq("1", t.acceptsValue(1));
    }

    dataTest() {
        AbstractDataProviderType t = AbstractDataProviderType::get(DataType);
        binary val = get_random_bytes(20);
        assertEq(val, t.acceptsValue(val));
        t = AbstractDataProviderType::get(DataOrNothingType);
        assertEq(val, t.acceptsValue(val));
    }

    compatTest() {
        QoreStringDataType str();
        HashDataType t1("t1");
        t1.setDefaultOtherFieldType();
        t1.addField(new QoreDataField("a", "desc", str));
        HashDataType t2("t2");
        t2.setDefaultOtherFieldType();
        t2.addField(new QoreDataField("b", "desc", str));
        HashDataType t3("t3");
        HashDeclDataType t4(TypedHash::forName("StatInfo"));
        HashDataType t5("t5");
        t5.addField(new QoreDataField("a", "desc", str));
        t5.addField(new QoreDataField("b", "desc", str));
        HashDataType t6("t6");
        t6.addField(new QoreDataField("a", "desc", str));
        HashDataType t7("t7");
        t7.addField(new QoreDataField("b", "desc", str));
        HashDataType t8("t8");
        t8.addField(new QoreDataField("type", "desc", str));

        assertFalse(t1.isAssignableFrom(t2));
        assertTrue(t3.isAssignableFrom(t1));
        assertTrue(t3.isAssignableFrom(t2));
        assertTrue(t3.isAssignableFrom(t4));
        assertTrue(t1.isAssignableFrom(t3));
        assertTrue(t2.isAssignableFrom(t3));
        assertFalse(t1.isAssignableFrom(t5));
        assertFalse(t2.isAssignableFrom(t5));

        assertFalse(t1.isAssignableFrom(str));
        assertFalse(t2.isAssignableFrom(str));
        assertFalse(t3.isAssignableFrom(str));

        assertFalse(t6.isAssignableFrom(t2));
        assertTrue(t3.isAssignableFrom(t6));
        assertTrue(t3.isAssignableFrom(t7));
        assertTrue(t6.isAssignableFrom(t3));
        assertTrue(t7.isAssignableFrom(t3));
        assertFalse(t6.isAssignableFrom(t5));
        assertFalse(t7.isAssignableFrom(t5));

        assertFalse(t6.isAssignableFrom(str));
        assertFalse(t7.isAssignableFrom(str));
        assertFalse(t3.isAssignableFrom(str));

        assertFalse(t4.isAssignableFrom(t1));
        assertFalse(t4.isAssignableFrom(t8));
        assertFalse(t8.isAssignableFrom(t4));

        {
            hash<auto> v = {"type": "str"};
            assertEq(v, t8.acceptsValue(v));
            assertEq(v, t4.acceptsValue(v));
        }

        hash<auto> h1;
        h1 += {
            "a": "string",
        };
        hash<auto> h2;
        h2 += {
            "b": "string",
        };
        assertThrows("RUNTIME-TYPE-ERROR", \t1.acceptsValue(), h2);
        assertThrows("RUNTIME-TYPE-ERROR", \t2.acceptsValue(), h1);
        assertThrows("RUNTIME-TYPE-ERROR", \t1.acceptsValue(), 1);
        assertThrows("RUNTIME-TYPE-ERROR", \t2.acceptsValue(), 1);
        assertThrows("RUNTIME-TYPE-ERROR", \t3.acceptsValue(), 1);
        assertEq(h1, t1.acceptsValue(h1));
        assertEq(h2, t2.acceptsValue(h2));
        assertEq(h1, t3.acceptsValue(h1));
        assertEq(h2, t3.acceptsValue(h2));
        assertEq(<StatInfo>{"size": 1}, t3.acceptsValue(<StatInfo>{"size": 1}));
        assertThrows("RUNTIME-TYPE-ERROR", \t1.acceptsValue(), h2 + {"a": "string"});
        assertThrows("RUNTIME-TYPE-ERROR", \t2.acceptsValue(), h1 + {"b": "string"});
        assertThrows("RUNTIME-TYPE-ERROR", \t1.acceptsValue(), {"a": 1});
        assertThrows("RUNTIME-TYPE-ERROR", \t2.acceptsValue(), {"b": 2});
        t1.setDefaultOtherFieldType(new QoreIntDataType());
        t2.setDefaultOtherFieldType(new QoreIntDataType());
        h1 += {"b": 2};
        h2 += {"a": 2};
        assertEq(h1, t1.acceptsValue(h1));
        assertEq(h2, t2.acceptsValue(h2));
    }

    pipelineTest() {
        {
            TestRecordProvider record_provider();
            DataProviderPipeline pipe();
            assertThrows("PIPELINE-ERROR", \pipe.submit(), {});

            TestRecordProcessor record_processor0("A");
            assertEq(AbstractDataProviderType::anyDataType, record_processor0.getExpectedType());
            assertEq(AbstractDataProviderType::anyDataType, record_processor0.getReturnType());
            pipe.append(record_processor0);
            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq({"A-a": 1, "A-b": 2}, record_processor0.recs[0]);

            hash<PipelineInfo> info = pipe.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            TestRecordProcessor record_processor1("B");
            pipe.append(record_processor1);
            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processor1.recs[0]);

            info = pipe.getInfo();
            assertEq(2, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipeline pipe1 = pipe.copy();
            record_processor0.clear();
            record_processor1.clear();

            pipe1.submit(record_provider);
            pipe1.waitDone();
            assertEq({"A-a": 1, "A-b": 2}, record_processor0.recs[0]);
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processor1.recs[0]);

            info = pipe1.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipelineFactory factory();
            factory.append(record_processor0);
            factory.append(record_processor1);
            assertEq({
                "opts": NOTHING,
                "num_queues": 1,
                "bulk": True,
            }, factory.getInfo());

            DataProviderPipeline pipe2 = factory.create();
            record_processor0.clear();
            record_processor1.clear();

            pipe2.submit(record_provider);
            pipe2.waitDone();
            assertEq({"A-a": 1, "A-b": 2}, record_processor0.recs[0]);
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processor1.recs[0]);

            info = pipe2.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);
        }

        {
            TestRecordProvider record_provider();
            DataProviderPipeline pipe(<PipelineOptionInfo>{
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });
            assertThrows("PIPELINE-ERROR", \pipe.submit(), {});

            hash<auto> flush_rec = {
                "x-1": get_random_string(),
            };
            TestFlushRecordProcessor record_processor0("A", NOTHING, NOTHING, flush_rec);
            assertEq(AbstractDataProviderType::anyDataType, record_processor0.getExpectedType());
            assertEq(AbstractDataProviderType::anyDataType, record_processor0.getReturnType());
            pipe.append(record_processor0);
            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq({"A-a": 1, "A-b": 2}, record_processor0.recs[0]);

            hash<PipelineInfo> info = pipe.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            TestRecordProcessor record_processor1("B");
            pipe.append(record_processor1);
            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processor1.recs[0]);

            info = pipe.getInfo();
            assertEq(2, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipeline pipe1 = pipe.copy();
            record_processor0.clear();
            record_processor1.clear();

            pipe1.submit(record_provider);
            pipe1.waitDone();
            assertEq({"A-a": 1, "A-b": 2}, record_processor0.recs[0]);
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processor1.recs[0]);
            assertEq({"B-x-1": flush_rec."x-1"}, record_processor1.recs[1]);

            info = pipe1.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipelineFactory factory();
            factory.append(record_processor0);
            factory.append(record_processor1);
            assertEq({
                "opts": NOTHING,
                "num_queues": 1,
                "bulk": True,
            }, factory.getInfo());

            DataProviderPipeline pipe2 = factory.create();
            record_processor0.clear();
            record_processor1.clear();

            pipe2.submit(record_provider);
            pipe2.waitDone();
            assertEq({"A-a": 1, "A-b": 2}, record_processor0.recs[0]);
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processor1.recs[0]);

            info = pipe2.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);
        }

        {
            TestRecordProvider record_provider();
            DataProviderPipeline pipe(<PipelineOptionInfo>{
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });

            TestRecordProcessor record_processorA0("A");
            TestRecordProcessor record_processorA1("B");

            TestRecordProcessor record_processorB0("A");
            TestRecordProcessor record_processorB1("B");

            # run pipeline in parallel
            int p0 = pipe.appendQueue(0);
            int p1 = pipe.appendQueue(0);

            pipe.append(p0, record_processorA0);
            pipe.append(p0, record_processorA1);

            pipe.append(p1, record_processorB1);
            pipe.append(p1, record_processorB0);

            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processorA1.recs[0]);
            assertEq({"A-B-a": 1, "A-B-b": 2}, record_processorB0.recs[0]);

            hash<PipelineInfo> info = pipe.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipeline pipe1 = pipe.copy();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();
            pipe1.submit(record_provider);
            pipe1.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processorA1.recs[0]);
            assertEq({"A-B-a": 1, "A-B-b": 2}, record_processorB0.recs[0]);

            info = pipe1.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipelineFactory factory();
            # run pipeline in parallel
            p0 = factory.appendQueue(0);
            p1 = factory.appendQueue(0);

            factory.append(p0, record_processorA0);
            factory.append(p0, record_processorA1);

            factory.append(p1, record_processorB1);
            factory.append(p1, record_processorB0);

            assertEq({
                "opts": NOTHING,
                "num_queues": 3,
                "bulk": True,
            }, factory.getInfo());

            DataProviderPipeline pipe2 = factory.create();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();
            pipe2.submit(record_provider);
            pipe2.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processorA1.recs[0]);
            assertEq({"A-B-a": 1, "A-B-b": 2}, record_processorB0.recs[0]);

            info = pipe2.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);
        }

        {
            TestRecordProvider record_provider();
            DataProviderPipeline pipe(<PipelineOptionInfo>{
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });

            hash<auto> flush_rec = {
                "x-1": get_random_string(),
            };
            TestFlushRecordProcessor record_processorA0("A", NOTHING, NOTHING, flush_rec);
            TestRecordProcessor record_processorA1("B");

            TestRecordProcessor record_processorB0("A");
            TestFlushRecordProcessor record_processorB1("B", NOTHING, NOTHING, flush_rec);

            # run pipeline in parallel
            int p0 = pipe.appendQueue(0);
            int p1 = pipe.appendQueue(0);

            pipe.append(p0, record_processorA0);
            pipe.append(p0, record_processorA1);

            pipe.append(p1, record_processorB1);
            pipe.append(p1, record_processorB0);

            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processorA1.recs[0]);
            assertEq({"A-B-a": 1, "A-B-b": 2}, record_processorB0.recs[0]);
            assertEq({"B-x-1": flush_rec."x-1"}, record_processorA1.recs[1]);
            assertEq({"A-x-1": flush_rec."x-1"}, record_processorB0.recs[1]);

            hash<PipelineInfo> info = pipe.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipeline pipe1 = pipe.copy();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();
            pipe1.submit(record_provider);
            pipe1.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processorA1.recs[0]);
            assertEq({"A-B-a": 1, "A-B-b": 2}, record_processorB0.recs[0]);

            info = pipe1.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);

            DataProviderPipelineFactory factory();
            # run pipeline in parallel
            p0 = factory.appendQueue(0);
            p1 = factory.appendQueue(0);

            factory.append(p0, record_processorA0);
            factory.append(p0, record_processorA1);

            factory.append(p1, record_processorB1);
            factory.append(p1, record_processorB0);

            assertEq({
                "opts": NOTHING,
                "num_queues": 3,
                "bulk": True,
            }, factory.getInfo());

            DataProviderPipeline pipe2 = factory.create();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();
            pipe2.submit(record_provider);
            pipe2.waitDone();
            assertEq({"B-A-a": 1, "B-A-b": 2}, record_processorA1.recs[0]);
            assertEq({"A-B-a": 1, "A-B-b": 2}, record_processorB0.recs[0]);

            info = pipe2.getInfo();
            assertEq(1, info.record_count);
            assertTrue(info.bulk);
        }

        {
            TestBulkRecordIterator record_provider();
            DataProviderPipeline pipe(<PipelineOptionInfo>{
                "name": "test-pipeline-3",
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });

            TestRecordProcessor record_processorA0("A", {"A0": 1});
            TestRecordProcessor record_processorA1("B", {"A1": 1});

            TestRecordProcessor record_processorB0("A", {"B0": 1});
            TestRecordProcessor record_processorB1("B", {"B1": 1});

            # run pipeline in parallel
            int p0 = pipe.appendQueue(0);
            int p1 = pipe.appendQueue(0);

            pipe.append(p0, record_processorA0);
            pipe.append(p0, record_processorA1);

            pipe.append(p1, record_processorB1);
            pipe.append(p1, record_processorB0);

            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq({"B-A-a": (1, 2), "B-A-b": ("one", "two"), "B-A0": 1, "A1": 1}, record_processorA1.recs[0]);
            assertEq({"A-B-a": (1, 2), "A-B-b": ("one", "two"), "A-B1": 1, "B0": 1}, record_processorB0.recs[0]);

            hash<PipelineInfo> info = pipe.getInfo();
            assertEq(2, info.record_count);
            assertTrue(info.bulk);

            pipe.submit(record_provider);
            pipe.waitDone();

            info = pipe.getInfo();
            assertEq(4, info.record_count);
            assertEq(PS_IDLE, info.status);

            pipe.reset();
            pipe.submit(record_provider);
            pipe.waitDone();

            info = pipe.getInfo();
            assertEq(PS_IDLE, info.status);
            assertEq(2, info.record_count);

            DataProviderPipeline pipe1 = pipe.copy();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();

            pipe1.submit(record_provider);
            pipe1.waitDone();
            assertEq({"B-A-a": (1, 2), "B-A-b": ("one", "two"), "B-A0": 1, "A1": 1}, record_processorA1.recs[0]);
            assertEq({"A-B-a": (1, 2), "A-B-b": ("one", "two"), "A-B1": 1, "B0": 1}, record_processorB0.recs[0]);

            info = pipe1.getInfo();
            assertEq(2, info.record_count);
            assertTrue(info.bulk);

            pipe1.submit(record_provider);
            pipe1.waitDone();

            info = pipe1.getInfo();
            assertEq(4, info.record_count);
            assertEq(PS_IDLE, info.status);

            pipe1.reset();
            pipe1.submit(record_provider);
            pipe1.waitDone();

            info = pipe1.getInfo();
            assertEq(PS_IDLE, info.status);
            assertEq(2, info.record_count);

            DataProviderPipelineFactory factory();
            # run pipeline in parallel
            p0 = factory.appendQueue(0);
            p1 = factory.appendQueue(0);

            factory.append(p0, record_processorA0);
            factory.append(p0, record_processorA1);

            factory.append(p1, record_processorB1);
            factory.append(p1, record_processorB0);

            assertEq({
                "opts": NOTHING,
                "num_queues": 3,
                "bulk": True,
            }, factory.getInfo());

            DataProviderPipeline pipe2 = factory.create();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();

            pipe2.submit(record_provider);
            pipe2.waitDone();
            assertEq({"B-A-a": (1, 2), "B-A-b": ("one", "two"), "B-A0": 1, "A1": 1}, record_processorA1.recs[0]);
            assertEq({"A-B-a": (1, 2), "A-B-b": ("one", "two"), "A-B1": 1, "B0": 1}, record_processorB0.recs[0]);

            info = pipe2.getInfo();
            assertEq(2, info.record_count);
            assertTrue(info.bulk);

            pipe2.submit(record_provider);
            pipe2.waitDone();

            info = pipe2.getInfo();
            assertEq(4, info.record_count);
            assertEq(PS_IDLE, info.status);

            pipe2.reset();
            pipe2.submit(record_provider);
            pipe2.waitDone();

            info = pipe2.getInfo();
            assertEq(PS_IDLE, info.status);
            assertEq(2, info.record_count);
        }

        # verify pipeline abort
        {
            InfiniteRecordProvider record_provider();
            hash<PipelineOptionInfo> opts({
                "name": "test-pipeline-3",
                #"info_log": sub (string fmt) { vprintf("TID " + gettid() + " " + fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf("TID " + gettid() + " " + fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf("TID " + gettid() + " " + fmt + "\n", argv); },
            });
            DataProviderPipeline pipe(opts);

            TestRecordProcessor record_processor0("A");
            pipe.append(record_processor0);
            hash<PipelineInfo> info = pipe.getInfo();
            assertNothing(info.start_time);
            assertNothing(info.stop_time);
            assertEq(PS_IDLE, info.status);
            assertEq(0, info.record_count);
            assertEq(1, info.num_queues);
            assertEq(True, info.bulk);
            assertEq(0s, info.duration);
            assertEq(0.0, info.duration_secs);
            assertEq(0.0, info.recs_per_sec);
            pipe.submit({"a": 1});
            assertEq(PS_RUNNING, pipe.getInfo().status);

            # wait for record count = 1
            while (True) {
                int size = record_processor0.recs.size();
                if (size == 1) {
                    break;
                }
                sleep(50ms);
            }

            Counter thread_cnt(1);
            Queue abort_queue();
            code submit = sub () {
                if (m_options.verbose > 2) {
                    printf("TID %d: infinite submission thread started\n", gettid());
                }
                on_exit if (m_options.verbose > 2) {
                    printf("TID %d: infinite submission thread exiting\n", gettid());
                }
                try {
                    thread_cnt.dec();
                    pipe.submit(record_provider);
                    abort_queue.push();
                } catch (hash<ExceptionInfo> ex) {
                    abort_queue.push(ex);
                }
            };
            background submit();
            thread_cnt.waitForZero();

            # issue #4317: do not abort before we know the infinite record provider is in progress
            # wait for record count >= 2
            while (True) {
                int size = record_processor0.recs.size();
                if (size >= 2) {
                    break;
                }
                sleep(50ms);
            }

            if (m_options.verbose > 2) {
                printf("TID %d: record processor0 size: %d; about to abort\n", gettid(), record_processor0.recs.size());
                hash<auto> stacks = get_all_thread_call_stacks();
                hash<auto> tmap;
                foreach hash<auto> i in (stacks.pairIterator()) {
                    splice i.value, 0, 2;
                    tmap{i.key} = map $1.type != "new-thread" ? sprintf("%s %s()", get_ex_pos($1), $1.function) : "new-thread", i.value;
                }
                printf("all threads: %N\n", tmap);
            }

            pipe.abort();
            assertEq(PS_ABORTED, pipe.getInfo().status);
            assertThrows("PIPELINE-ERROR", \pipe.submit(), {});

            if (m_options.verbose > 2) {
                printf("TID %d: pipeline info: %N\n", gettid(), pipe.getInfo());
            }

            *hash<ExceptionInfo> ex_info = abort_queue.get();
            assertEq("PIPELINE-SUBMISSION-ABORTED", ex_info.err);

            if (m_options.verbose > 2) {
                printf("TID %d: infinite submission exception: %s: %s\n", gettid(), ex_info.err, ex_info.desc);

            }

            pipe.reset();
            pipe.submit({"a": 1});
            pipe.waitDone();
            assertEq(PS_IDLE, pipe.getInfo().status);

            DataProviderPipeline pipe1 = pipe.copy();
            hash<PipelineInfo> info2 = pipe1.getInfo();
            assertEq(info - "name", info2 - "name");
            assertNothing(info2.start_time);
            assertNothing(info2.stop_time);
            assertEq(PS_IDLE, info2.status);
            assertEq(0, info2.record_count);
            assertEq(1, info2.num_queues);
            assertEq(True, info2.bulk);
            assertEq(0s, info2.duration);
            assertEq(0.0, info2.duration_secs);
            assertEq(0.0, info2.recs_per_sec);

            # clear record processor cache
            record_processor0.clear();

            pipe1.submit({"a": 1});
            assertEq(PS_RUNNING, pipe1.getInfo().status);

            # wait for record count = 1
            while (True) {
                int size = record_processor0.recs.size();
                if (size == 1) {
                    break;
                }
                sleep(50ms);
            }

            assertEq(0, thread_cnt.getCount());
            assertEq(0, abort_queue.size());
            thread_cnt.inc();
            submit = sub () {
                try {
                    thread_cnt.dec();
                    pipe1.submit(record_provider);
                    abort_queue.push();
                } catch (hash<ExceptionInfo> ex) {
                    abort_queue.push(ex);
                }
            };
            background submit();
            thread_cnt.waitForZero();

            # issue #4317: do not abort before we know the infinite record provider is in progress
            # wait for record count >= 2
            while (True) {
                int size = record_processor0.recs.size();
                if (size >= 2) {
                    break;
                }
                sleep(50ms);
            }

            pipe1.abort();
            assertEq(PS_ABORTED, pipe1.getInfo().status);
            #printf("%N\n", pipe.getInfo());
            assertThrows("PIPELINE-ERROR", \pipe1.submit(), {});

            ex_info = abort_queue.get();
            assertEq("PIPELINE-SUBMISSION-ABORTED", ex_info.err);

            pipe1.reset();
            pipe1.submit({"a": 1});
            pipe1.waitDone();
            assertEq(PS_IDLE, pipe1.getInfo().status);

            DataProviderPipelineFactory factory(opts);
            factory.append(record_processor0);
            assertEq({
                "opts": opts,
                "num_queues": 1,
                "bulk": True,
            }, factory.getInfo());

            DataProviderPipeline pipe2 = factory.create();
            info2 = pipe2.getInfo();
            assertEq(info, info2);
            assertNothing(info2.start_time);
            assertNothing(info2.stop_time);
            assertEq(PS_IDLE, info2.status);
            assertEq(0, info2.record_count);
            assertEq(1, info2.num_queues);
            assertEq(True, info2.bulk);
            assertEq(0s, info2.duration);
            assertEq(0.0, info2.duration_secs);
            assertEq(0.0, info2.recs_per_sec);

            # clear record processor cache
            record_processor0.clear();

            pipe2.submit({"a": 1});
            assertEq(PS_RUNNING, pipe2.getInfo().status);

            # wait for record count = 1
            while (True) {
                int size = record_processor0.recs.size();
                if (size == 1) {
                    break;
                }
                sleep(50ms);
            }

            assertEq(0, thread_cnt.getCount());
            assertEq(0, abort_queue.size());
            thread_cnt.inc();
            submit = sub () {
                try {
                    thread_cnt.dec();
                    pipe2.submit(record_provider);
                    abort_queue.push();
                } catch (hash<ExceptionInfo> ex) {
                    abort_queue.push(ex);
                }
            };
            background submit();
            thread_cnt.waitForZero();

            # issue #4317: do not abort before we know the infinite record provider is in progress
            # wait for record count >= 2
            while (True) {
                int size = record_processor0.recs.size();
                if (size >= 2) {
                    break;
                }
                sleep(50ms);
            }

            pipe2.abort();
            assertEq(PS_ABORTED, pipe2.getInfo().status);
            #printf("%N\n", pipe.getInfo());
            assertThrows("PIPELINE-ERROR", \pipe2.submit(), {});

            ex_info = abort_queue.get();
            assertEq("PIPELINE-SUBMISSION-ABORTED", ex_info.err);

            pipe2.reset();
            pipe2.submit({"a": 1});
            pipe2.waitDone();
            assertEq(PS_IDLE, pipe2.getInfo().status);
        }

        # verify pipeline error handling
        {
            TestBulkRecordIterator record_provider();
            hash<PipelineOptionInfo> opts({
                "name": "test-pipeline-4",
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });
            DataProviderPipeline pipe(opts);

            TestRecordProcessor record_processorA0("A", {"A0": 1});
            TestRecordProcessor record_processorA1("B", {"A1": 1});

            TestRecordProcessor record_processorB0("A", {"B0": 1});
            TestRecordProcessor record_processorB1("B", {"B1": 1}, True);

            # run pipeline in parallel
            int p0 = pipe.appendQueue(0);
            int p1 = pipe.appendQueue(0);

            pipe.append(p0, record_processorA0);
            pipe.append(p0, record_processorA1);

            pipe.append(p1, record_processorB1);
            pipe.append(p1, record_processorB0);

            pipe.submit(record_provider);
            assertThrows("PIPELINE-FAILED", \pipe.waitDone());
            assertEq(PS_ABORTED, pipe.getInfo().status);
            pipe.reset();
            pipe.submit({"a": 1});
            pipe.waitDone();
            assertEq(PS_IDLE, pipe.getInfo().status);

            DataProviderPipeline pipe1 = pipe.copy();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();
            record_processorB1.setRaiseError();

            pipe1.submit(record_provider);
            assertThrows("PIPELINE-FAILED", \pipe1.waitDone());
            assertEq(PS_ABORTED, pipe1.getInfo().status);
            pipe1.reset();
            pipe1.submit({"a": 1});
            pipe1.waitDone();
            assertEq(PS_IDLE, pipe1.getInfo().status);

            DataProviderPipelineFactory factory(opts);

            # run pipeline in parallel
            p0 = factory.appendQueue(0);
            p1 = factory.appendQueue(0);

            factory.append(p0, record_processorA0);
            factory.append(p0, record_processorA1);

            factory.append(p1, record_processorB1);
            factory.append(p1, record_processorB0);

            assertEq({
                "opts": opts,
                "num_queues": 3,
                "bulk": True,
            }, factory.getInfo());

            DataProviderPipeline pipe2 = factory.create();
            record_processorA0.clear();
            record_processorA1.clear();
            record_processorB0.clear();
            record_processorB1.clear();
            record_processorB1.setRaiseError();

            pipe2.submit(record_provider);
            assertThrows("PIPELINE-FAILED", \pipe2.waitDone());
            assertEq(PS_ABORTED, pipe2.getInfo().status);
            pipe2.reset();
            pipe2.submit({"a": 1});
            pipe2.waitDone();
            assertEq(PS_IDLE, pipe2.getInfo().status);
        }

        # test non-record-based pipeline processing
        {
            hash<PipelineOptionInfo> opts({
                "name": "test-pipeline-5",
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });
            DataProviderPipeline pipe(opts);
            TestIntProcessor int_processor0(1);
            TestIntProcessor int_processor1(2);

            # run pipeline in parallel
            int q0 = pipe.appendQueue(0);
            int q1 = pipe.appendQueue(0);

            pipe.append(q0, int_processor0);
            pipe.append(q0, int_processor0);

            pipe.append(q1, int_processor1);
            pipe.append(q1, int_processor1);

            hash<PipelineInfo> info = pipe.getInfo();

            pipe.submit(1);
            pipe.waitDone();
            assertEq(5, int_processor0.getSum());
            assertEq(8, int_processor1.getSum());

            DataProviderPipeline pipe1 = pipe.copy();
            hash<PipelineInfo> info1 = pipe1.getInfo();
            assertEq(info - "name", info1 - "name");

            int_processor0.reset();
            int_processor1.reset();

            pipe1.submit(1);
            pipe1.waitDone();
            assertEq(5, int_processor0.getSum());
            assertEq(8, int_processor1.getSum());

            DataProviderPipelineFactory factory(opts);
            # run pipeline in parallel
            q0 = factory.appendQueue(0);
            q1 = factory.appendQueue(0);

            factory.append(q0, int_processor0);
            factory.append(q0, int_processor0);

            factory.append(q1, int_processor1);
            factory.append(q1, int_processor1);

            DataProviderPipeline pipe2 = factory.create();
            hash<PipelineInfo> info2 = pipe2.getInfo();
            assertEq(info, info2);

            int_processor0.reset();
            int_processor1.reset();

            pipe2.submit(1);
            pipe2.waitDone();
            assertEq(5, int_processor0.getSum());
            assertEq(8, int_processor1.getSum());
        }

        # test data skipping
        {
            hash<PipelineOptionInfo> opts({
                "name": "test-pipeline-6",
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });
            DataProviderPipeline pipe(opts);
            TestIntProcessor int_processor0(1);
            TestIntProcessor int_processor1(2, True);

            # run pipeline in parallel
            int q0 = pipe.appendQueue(0);
            int q1 = pipe.appendQueue(0);

            pipe.append(q0, int_processor0);
            pipe.append(q0, int_processor0);

            pipe.append(q1, int_processor1);
            pipe.append(q1, int_processor1);

            pipe.submit(1);
            pipe.waitDone();
            assertEq(5, int_processor0.getSum());
            assertEq(3, int_processor1.getSum());

            DataProviderPipeline pipe1 = pipe.copy();
            int_processor0.reset();
            int_processor1.reset();

            pipe1.submit(1);
            pipe1.waitDone();
            assertEq(5, int_processor0.getSum());
            assertEq(3, int_processor1.getSum());

            int count = 0;
            code thread_callback = sub () { ++count; };
            opts.thread_callback = thread_callback;
            DataProviderPipelineFactory factory(opts);

            # run pipeline in parallel
            q0 = factory.appendQueue(0);
            q1 = factory.appendQueue(0);

            factory.append(q0, int_processor0);
            factory.append(q0, int_processor0);

            factory.append(q1, int_processor1);
            factory.append(q1, int_processor1);

            DataProviderPipeline pipe2 = factory.create();
            int_processor0.reset();
            int_processor1.reset();

            pipe2.submit(1);
            pipe2.waitDone();
            assertEq(5, int_processor0.getSum());
            assertEq(3, int_processor1.getSum());
            assertEq(3, count);
        }

        {
            hash<PipelineOptionInfo> opts({
                "name": "test-pipeline-7",
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });
            DataProviderPipeline pipe(opts);
            DuplicateRecordProcessor proc1("x", "y");
            pipe.append(proc1);
            RecordProcessorSink proc2();
            pipe.append(proc2);
            TestRecordProvider record_provider();
            pipe.submit(record_provider);
            pipe.waitDone();
            assertEq((
                {
                    "x-a": 1,
                    "x-b": 2,
                },
                {
                    "y-a": 1,
                    "y-b": 2,
                },
            ), proc2.recs);
        }

        {
            hash<PipelineOptionInfo> opts({
                "name": "test-pipeline-8",
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });
            DataProviderPipeline pipe(opts);
            DummyRecordProcessor proc1();
            pipe.append(proc1);
            RecordProcessorSink proc2();
            pipe.append(proc2);
            pipe.submit();
            pipe.waitDone();
            assertEq(({"x": 1,},), proc2.recs);
        }

        {
            hash<PipelineOptionInfo> opts({
                "name": "test-pipeline-9",
                #"info_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"debug_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
                #"error_log": sub (string fmt) { vprintf(fmt + "\n", argv); },
            });
            DataProviderPipeline pipe(opts);
            RecordProcessorSink proc1();
            pipe.append(proc1);
            RecordProcessorSink proc2();
            pipe.append(proc2);
            list<auto> l = (0, "one");
            pipe.submit(l);
            pipe.waitDone();
            assertEq((l,), proc1.recs);
            assertEq((l,), proc2.recs);
        }
    }

    typeCacheTest() {
        HashDataType h1();
        HashDataType h2();
        HashDataType h3();
        DataProviderTypeCache type_cache();
        assertEq(0, type_cache.size());
        assertTrue(type_cache.empty());

        type_cache.registerType("test/one", h1);
        assertEq(1, type_cache.size());
        assertFalse(type_cache.empty());
        assertThrows("TYPE-ERROR", \type_cache.registerType(), ("test/one", h2));
        hash<DataProviderTypeEntryInfo> type_info = type_cache.getTypeInfo("test/one");
        assertEq("one", type_info.name);
        assertEq("/test/one", type_info.path);
        assertFalse(type_info.locked);
        assertTrue(type_info.has_type);
        assertNothing(type_info.children);
        type_info = type_cache.getTypeInfo("/test/one");
        assertEq("one", type_info.name);
        assertEq("/test/one", type_info.path);
        assertFalse(type_info.locked);
        assertTrue(type_info.has_type);
        assertNothing(type_info.children);

        type_cache.registerType("test/two", h2);
        assertEq(2, type_cache.size());
        assertThrows("TYPE-ERROR", \type_cache.registerType(), ("test/two", h1));
        type_info = type_cache.getTypeInfo("test/two");
        assertEq("two", type_info.name);
        assertEq("/test/two", type_info.path);
        assertFalse(type_info.locked);
        assertTrue(type_info.has_type);
        assertNothing(type_info.children);
        type_info = type_cache.getTypeInfo("/test/two");
        assertEq("two", type_info.name);
        assertEq("/test/two", type_info.path);
        assertFalse(type_info.locked);
        assertTrue(type_info.has_type);
        assertNothing(type_info.children);

        type_cache.registerType("test/three", h3, True);
        assertEq(3, type_cache.size());
        assertThrows("TYPE-ERROR", \type_cache.registerType(), ("test/three", h1));
        assertEq(("/test/one", "/test/two", "/test/three"), type_cache.listTypes());
        assertEq(h1, type_cache.getType("test/one"));
        assertEq(h1, type_cache.getType("/test/one"));
        assertEq(h2, type_cache.getType("test/two"));
        assertEq(h3, type_cache.getType("test/three"));
        assertNothing(type_cache.getType("test/xxx"));
        assertNothing(type_cache.getType("/test/xxx"));
        assertThrows("PROVIDER-ERROR", \type_cache.getTypeEx(), ("test/xxx"));
        assertThrows("PROVIDER-ERROR", \type_cache.getTypeEx(), ("/test/xxx"));
        type_info = type_cache.getTypeInfo("test/three");
        assertEq("three", type_info.name);
        assertEq("/test/three", type_info.path);
        assertTrue(type_info.locked);
        assertTrue(type_info.has_type);
        assertNothing(type_info.children);
        type_info = type_cache.getTypeInfo("/test/three");
        assertEq("three", type_info.name);
        assertEq("/test/three", type_info.path);
        assertTrue(type_info.locked);
        assertTrue(type_info.has_type);
        assertNothing(type_info.children);

        HashDataType h2bis();
        assertTrue(type_cache.registerOrReplaceType("test/two", h2bis));
        assertEq(3, type_cache.size());
        assertEq(h2bis, type_cache.getType("test/two"));
        assertEq(h2bis, type_cache.removeType("test/two"));
        assertEq(2, type_cache.size());
        assertNothing(type_cache.getType("test/two"));
        assertNothing(type_cache.getType("/test/two"));
        assertThrows("PROVIDER-ERROR", \type_cache.getTypeEx(), ("test/two"));
        assertThrows("PROVIDER-ERROR", \type_cache.getTypeEx(), ("/test/two"));
        assertEq(("/test/one", "/test/three"), type_cache.listTypes());
        assertEq(h1, type_cache.getType("test/one"));
        assertEq(h1, type_cache.getType("/test/one"));
        assertEq(h3, type_cache.getType("test/three"));
        assertEq(h3, type_cache.getType("/test/three"));
        assertFalse(type_cache.registerOrReplaceType("test/one", h1));
        assertFalse(type_cache.registerOrReplaceType("/test/one", h1));
        type_cache.registerType("test/one", h1);
        type_cache.registerType("/test/one", h1);
        assertEq(2, type_cache.size());
        type_cache.clear();
        assertEq(2, type_cache.size());
        assertEq(("/test/three",), type_cache.listTypes());

        DataProviderTypeCache t2 = Serializable::deserialize(type_cache.serialize());
        assertEq(2, t2.size());

        HashDataType h4();
        h4.addField(new QoreDataField("key1", "desc1", StringType));
        HashDataType ch1();
        ch1.addField(new QoreDataField("key1", "desc1", StringType));
        h4.addField(new QoreDataField("key2", "desc2", ch1));
        type_cache.registerType("test/four", h4);
        assertEq("string", type_cache.getType("/test/four/key1").getBaseTypeName());
        DataProviderTypeEntry entry = type_cache.getRoot().getChild("test").getChild("four");
        assertEq(("key1", "key2"), entry.getInfo().children);
        assertEq(("key2",), entry.getInfo(True).children);
        assertEq(("key1", "key2"), type_cache.getTypeInfo("/test/four").children);
        assertEq(("key2",), type_cache.getTypeInfo("/test/four", True).children);
        assertEq(("key1", "key2"), type_cache.getTypeInfoEx("/test/four").children);
        assertEq(("key2",), type_cache.getTypeInfoEx("/test/four", True).children);
    }

    dataProviderTest() {
        {
            assertEq("hash<auto>", AbstractDataProviderType::get("hash<auto>").getName());
            assertEq("hash<string, bool>", AbstractDataProviderType::get("hash<string, bool>").getName());
            assertEq("*hash<auto>", AbstractDataProviderType::get("*hash<auto>").getName());
            assertEq("*hash<string, bool>", AbstractDataProviderType::get("*hash<string, bool>").getName());
            assertThrows("UNKNOWN-TYPE", "xyz", \AbstractDataProviderType::get(), "xyz");
            assertThrows("UNKNOWN-TYPE", "hash<string>", \AbstractDataProviderType::get(), "hash<string>");
        }

        {
            AbstractDataProviderType type = DataProvider::getType("/qore/ftp/event");
            assertEq(AutoHashType, type.getValueType());
            type = DataProvider::getType("qore/ftp/event");
            {
                AbstractDataProviderType t = Serializable::deserialize(type.serialize());
                assertEq("FtpPollerFileEventInfoDataType", t.className());
                assertEq("data", t.getField("data").getType().getValueType().getName());
            }
            assertEq(AutoHashType, type.getValueType());
            assertNothing(DataProvider::getType("xxx"));
            assertThrows("PROVIDER-ERROR", \DataProvider::getTypeEx(), "xxx");
            type = DataProvider::getType("/qore/ftp/event/filepath");
            assertEq("string", type.getName());
            type = DataProvider::getType("qore/ftp/event/filepath");
            assertEq("string", type.getName());
            assertThrows("PROVIDER-ERROR", \DataProvider::getTypeEx(), "/qore/ftp/event/xxx");
            assertThrows("PROVIDER-ERROR", \DataProvider::getTypeEx(), "qore/ftp/event/xxx");
        }

        {
            Type type("hash<StatInfo>");
            AbstractDataProviderType datatype = AbstractDataProviderType::get(type);
            assertEq("hash<StatInfo>", datatype.getName());
            assertTrue(datatype.isAssignableFrom(HashType));
            assertTrue(IntType.isEqual(datatype.getField("size").getType().getValueType()));
        }

        {
            QoreNumberDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NOTHING);
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NULL);

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("softnumber", type1.getName());
            assertEq(1n, type1.acceptsValue("1"));
            assertEq(1234.56n, type1.acceptsValue("1234.56"));
        }

        {
            QoreNumberOrNothingDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("*softnumber", type1.getName());
            assertEq(1n, type1.acceptsValue("1"));
            assertEq(1234.56n, type1.acceptsValue("1234.56"));
            assertNothing(type1.acceptsValue());
            assertNothing(type1.acceptsValue(NULL));
        }

        {
            QoreSoftNumberDataType num1({"number.format": ".,"});
            assertEq(1234.56n, num1.acceptsValue("1.234,56"));
            assertThrows("RUNTIME-TYPE-ERROR", \num1.acceptsValue());
            assertEq(0n, num1.acceptsValue(NULL));
        }

        {
            QoreSoftNumberDataType num1({"qore.no_null": True});
            assertThrows("RUNTIME-TYPE-ERROR", \num1.acceptsValue());
            assertThrows("RUNTIME-TYPE-ERROR", \num1.acceptsValue(), NULL);
        }

        {
            QoreSoftNumberOrNothingDataType num2({"number.format": ".,"});
            assertEq(1234.56n, num2.acceptsValue("1.234,56"));
            assertNothing(num2.acceptsValue());
            assertNothing(num2.acceptsValue(NULL));
        }

        {
            QoreSoftNumberOrNothingDataType num2();
            assertNothing(num2.acceptsValue(""));
            assertNothing(num2.acceptsValue());
            assertNothing(num2.acceptsValue(NULL));
        }

        {
            QoreBinaryDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NOTHING);
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NULL);

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("softbinary", type1.getName());
            assertEq(<31>, type1.acceptsValue("1"));
            assertEq(<3132>, type1.acceptsValue("12"));
        }

        {
            QoreBinaryOrNothingDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("*softbinary", type1.getName());
            assertEq(<31>, type1.acceptsValue("1"));
            assertEq(<3132>, type1.acceptsValue("12"));
            assertNothing(type1.acceptsValue());
            assertNothing(type1.acceptsValue(NULL));
        }

        {
            QoreSoftBinaryDataType bin1();
            assertEq(<3132>, bin1.acceptsValue("12"));
            assertThrows("RUNTIME-TYPE-ERROR", \bin1.acceptsValue());
            assertEq(binary(), bin1.acceptsValue(NULL));
        }

        {
            QoreSoftBinaryDataType bin1({"qore.no_null": True});
            assertThrows("RUNTIME-TYPE-ERROR", \bin1.acceptsValue());
            assertThrows("RUNTIME-TYPE-ERROR", \bin1.acceptsValue(), NULL);
        }

        {
            QoreSoftBinaryOrNothingDataType bin1();
            assertEq(<3132>, bin1.acceptsValue("12"));
            assertNothing(bin1.acceptsValue(""));
            assertNothing(bin1.acceptsValue());
            assertNothing(bin1.acceptsValue(NULL));
        }

        {
            QoreIntDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NOTHING);
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NULL);

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("softint", type1.getName());
            assertEq(1, type1.acceptsValue("1"));
            assertEq(1234, type1.acceptsValue("1234"));
        }

        {
            QoreIntOrNothingDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("*softint", type1.getName());
            assertEq(1, type1.acceptsValue("1"));
            assertEq(1234, type1.acceptsValue("1234"));
            assertNothing(type1.acceptsValue());
            assertNothing(type1.acceptsValue(NULL));
        }

        {
            QoreSoftIntDataType int1();
            assertEq(1234, int1.acceptsValue("1234"));
            assertThrows("RUNTIME-TYPE-ERROR", \int1.acceptsValue());
            assertEq(0, int1.acceptsValue(NULL));
        }

        {
            QoreSoftIntDataType int1({"qore.no_null": True});
            assertEq(1234, int1.acceptsValue("1234"));
            assertThrows("RUNTIME-TYPE-ERROR", \int1.acceptsValue());
            assertThrows("RUNTIME-TYPE-ERROR", \int1.acceptsValue(), NULL);
        }

        {
            QoreSoftIntOrNothingDataType int2();
            assertEq(1234, int2.acceptsValue("1234"));
            assertNothing(int2.acceptsValue(""));
            assertNothing(int2.acceptsValue());
            assertNothing(int2.acceptsValue(NULL));
        }

        {
            QoreBoolDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NOTHING);
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NULL);

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("softbool", type1.getName());
            assertEq(True, type1.acceptsValue("1"));
            assertEq(True, type1.acceptsValue("1234"));
        }

        {
            QoreBoolOrNothingDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("*softbool", type1.getName());
            assertEq(True, type1.acceptsValue("1"));
            assertEq(True, type1.acceptsValue("1234"));
            assertNothing(type1.acceptsValue());
            assertNothing(type1.acceptsValue(NULL));
        }

        {
            QoreBoolDataType bool1();
            assertThrows("RUNTIME-TYPE-ERROR", \bool1.acceptsValue(), "1");
            assertThrows("RUNTIME-TYPE-ERROR", \bool1.acceptsValue(), NOTHING);
            assertThrows("RUNTIME-TYPE-ERROR", \bool1.acceptsValue(), NULL);
        }

        {
            QoreSoftBoolDataType bool1();
            assertEq(True, bool1.acceptsValue("1234"));
            assertThrows("RUNTIME-TYPE-ERROR", \bool1.acceptsValue());
            assertEq(False, bool1.acceptsValue(NULL));
        }

        {
            QoreSoftBoolDataType bool1({"qore.no_null": True});
            assertEq(True, bool1.acceptsValue("1234"));
            assertThrows("RUNTIME-TYPE-ERROR", \bool1.acceptsValue());
            assertThrows("RUNTIME-TYPE-ERROR", \bool1.acceptsValue(), NULL);
        }

        {
            QoreSoftBoolOrNothingDataType bool1();
            assertEq(True, bool1.acceptsValue("1234"));
            assertNothing(bool1.acceptsValue(""));
            assertNothing(bool1.acceptsValue());
            assertNothing(bool1.acceptsValue(NULL));
        }

        {
            QoreFloatDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NOTHING);
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NULL);

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("softfloat", type1.getName());
            assertEq(1.0, type1.acceptsValue("1"));
            assertEq(1234.1, type1.acceptsValue("1234.1"));
        }

        {
            QoreFloatOrNothingDataType type();
            assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");

            AbstractDataProviderType type1 = type.getSoftType();
            assertEq("*softfloat", type1.getName());
            assertEq(1.0, type1.acceptsValue("1"));
            assertEq(1234.1, type1.acceptsValue("1234.1"));
            assertNothing(type1.acceptsValue());
            assertNothing(type1.acceptsValue(NULL));
        }

        {
            QoreSoftFloatDataType float1();
            assertEq(1234.0, float1.acceptsValue("1234"));
            assertThrows("RUNTIME-TYPE-ERROR", \float1.acceptsValue());
            assertEq(0.0, float1.acceptsValue(NULL));
        }

        {
            QoreSoftFloatDataType float1({"qore.no_null": True});
            assertEq(1234.0, float1.acceptsValue("1234"));
            assertThrows("RUNTIME-TYPE-ERROR", \float1.acceptsValue());
            assertThrows("RUNTIME-TYPE-ERROR", \float1.acceptsValue(), NULL);
        }

        {
            QoreSoftFloatOrNothingDataType float1();
            assertEq(1234.0, float1.acceptsValue("1234"));
            assertNothing(float1.acceptsValue(""));
            assertNothing(float1.acceptsValue());
            assertNothing(float1.acceptsValue(NULL));
        }

        {
            hash<auto> date_opts({
                "date.format": "DD.MM.YYYY",
                "date.input_timezone": new TimeZone("America/Chicago"),
                "date.output_timezone": new TimeZone("UTC"),
            });

            {
                QoreDateDataType type();
                assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");
                assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NOTHING);
                assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), NULL);

                AbstractDataProviderType type1 = type.getSoftType();
                assertEq("softdate", type1.getName());
                assertEq(2019-11-09, type1.acceptsValue("2019-11-09"));
            }

            {
                QoreDateOrNothingDataType type();
                assertThrows("RUNTIME-TYPE-ERROR", \type.acceptsValue(), "1");

                AbstractDataProviderType type1 = type.getSoftType();
                assertEq("*softdate", type1.getName());
                assertEq(2019-11-09, type1.acceptsValue("2019-11-09"));
                assertNothing(type1.acceptsValue());
                assertNothing(type1.acceptsValue(NULL));
            }

            {
                QoreSoftDateDataType date1(date_opts);
                assertEq(2019-09-13T05:00:00Z, date1.acceptsValue("13.09.2019"));
                assertEq(1970-01-01Z, date1.acceptsValue(0));
                assertThrows("RUNTIME-TYPE-ERROR", \date1.acceptsValue());
                assertEq(1970-01-01, date1.acceptsValue(NULL));
            }

            {
                QoreSoftDateDataType date1({"qore.no_null": True});
                assertThrows("RUNTIME-TYPE-ERROR", \date1.acceptsValue());
                assertThrows("RUNTIME-TYPE-ERROR", \date1.acceptsValue(), NULL);
            }

            {
                QoreSoftDateOrNothingDataType date2(date_opts);
                assertEq(2019-09-13T05:00:00Z, date2.acceptsValue("13.09.2019"));
                assertNothing(date2.acceptsValue());
                assertNothing(date2.acceptsValue(NULL));
            }
        }

        {
            QoreSoftStringDataType str1();
            assertEq("123", str1.acceptsValue(123));
            assertThrows("RUNTIME-TYPE-ERROR", \str1.acceptsValue());
            assertEq("", str1.acceptsValue(NULL));
        }

        {
            QoreSoftStringDataType str({"qore.no_null": True});
            assertThrows("RUNTIME-TYPE-ERROR", \str.acceptsValue(), NULL);
        }

        {
            QoreSoftStringDataType str({"string.encoding": "iso-8859-1"});
            assertEq("ISO-8859-1", str.acceptsValue("hi").encoding());
            assertEq("", str.acceptsValue(""));
            assertThrows("RUNTIME-TYPE-ERROR", \str.acceptsValue());
        }

        {
            QoreStringDataType str1();
            assertThrows("RUNTIME-TYPE-ERROR", \str1.acceptsValue(), 123);
            assertThrows("RUNTIME-TYPE-ERROR", \str1.acceptsValue());
            assertThrows("RUNTIME-TYPE-ERROR", \str1.acceptsValue(), NULL);

            AbstractDataProviderType type1 = str1.getSoftType();
            assertEq("softstring", type1.getName());
            assertEq("123", type1.acceptsValue(123));
        }

        {
            QoreStringOrNothingDataType str2();
            assertThrows("RUNTIME-TYPE-ERROR", \str2.acceptsValue(), 123);
            assertNothing(str2.acceptsValue());
            assertNothing(str2.acceptsValue(NULL));

            AbstractDataProviderType type1 = str2.getSoftType();
            assertEq("*softstring", type1.getName());
            assertEq("123", type1.acceptsValue(123));
            assertNothing(type1.acceptsValue());
            assertNothing(type1.acceptsValue(NULL));
        }

        {
            QoreStringOrNothingDataType str({"string.empty_to_nothing": True});
            assertEq("UTF-8", str.acceptsValue("hi").encoding());
            assertNothing(str.acceptsValue(""));
        }

        {
            QoreSoftStringOrNothingDataType str2();
            assertEq("123", str2.acceptsValue(123));
            assertNothing(str2.acceptsValue());
            assertNothing(str2.acceptsValue(NULL));
        }

        {
            QoreHashDataType hash1(new Type("hash<string, bool>"));
            assertEq({"a": True}, hash1.acceptsValue({"a": True}));
            assertTrue(BoolType.isEqual(hash1.getFieldType("xxx").getValueType()));
            assertThrows("RUNTIME-TYPE-ERROR", \hash1.acceptsValue(), {"a": 1});
            assertThrows("RUNTIME-TYPE-ERROR", \hash1.acceptsValue());
        }

        {
            QoreHashDataType hash2(new Type("*hash<string, bool>"));
            assertTrue(BoolType.isEqual(hash2.getFieldType("xxx").getValueType()));
            assertThrows("RUNTIME-TYPE-ERROR", \hash2.acceptsValue(), {"a": 1});
            assertNothing(hash2.acceptsValue());
            assertNothing(hash2.acceptsValue(NULL));

            hash<DataTypeInfo> info = hash2.getInfo();
            assertEq("*hash<string, bool>", info.name);
            assertTrue(info.hasKey("supported_options"));
            assertTrue(info.hasKey("options"));
            assertNothing(info.supported_options);
            assertNothing(info.options);
            assertEq(3, info.types_accepted.size());
            assertEq(2, info.types_returned.size());

            info = hash2.getInputInfo();
            assertEq("*hash<string, bool>", info.name);
            assertFalse(info.hasKey("supported_options"));
            assertFalse(info.hasKey("options"));
            assertFalse(info.hasKey("types_accepted"));
            assertEq(2, info.types_returned.size());
        }

        {
            HashDataType fields();
            fields.addField(new QoreDataField("key1", "desc1", StringType));
            assertTrue(fields.getFields().key1 instanceof QoreDataField);
            assertEq("key1", fields.getFields().key1.getName());
            assertEq("desc1", fields.getFields().key1.getDescription());
            assertEq("key1", fields.getFields().key1.getInputInfo().name);
            assertEq("string", fields.getFields().key1.getInputInfo().type.name);
            assertEq("key1", fields.getFields().key1.getInfo().name);
            assertEq("string", fields.getFields().key1.getInfo().type.name);
            assertThrows("RUNTIME-TYPE-ERROR", \fields.getFields().key1.acceptsValue(), 1);

            HashDataType type1 = fields.getSoftType();
            assertTrue(type1.getFields().key1 instanceof QoreDataField);
            assertEq("key1", type1.getFields().key1.getName());
            assertEq("1", type1.getFields().key1.acceptsValue(1));
        }

        {
            HashDeclDataType type0(TypedHash::forName("StatInfo"));
            assertTrue(type0.getFields().size instanceof QoreDataField);

            AbstractDataProviderType type1 = type0.getSoftType();
            assertTrue(type1.getFields().size instanceof QoreDataField);
            assertEq("size", type1.getFields().size.getName());
            assertEq(1, type1.getFields().size.acceptsValue("1"));

            AbstractDataProviderType type2 = type0.getOrNothingType();
            assertTrue(type2.getFields().size instanceof QoreDataField);
            assertEq("size", type2.getFields().size.getName());
            assertNothing(type2.getFields().size.acceptsValue());
        }

        {
            QoreDataType t(SoftStringType);
            assertEq("", t.acceptsValue(NULL));
        }

        {
            QoreDataType t(SoftStringType, {"qore.no_null": True});
            assertThrows("RUNTIME-TYPE-ERROR", "does not accept.*NULL", \t.acceptsValue(), (NULL,));
        }

        {
            ListDataType t(IntType);
            assertEq("list<int>", t.getName());
            assertEq("list<int>", t.getValueType().getName());
            assertThrows("RUNTIME-TYPE-ERROR", "list.*integer", \t.acceptsValue(), 1);
            assertThrows("RUNTIME-TYPE-ERROR", "expects.*got type 'integer'", \t.acceptsValue(), (1,));
            assertThrows("RUNTIME-TYPE-ERROR", "int.*string", \t.acceptsValue(), (("a",),));
            assertThrows("RUNTIME-TYPE-ERROR", "expects.*got type 'NULL'", \t.acceptsValue(), (NULL,));
            assertEq((1,), t.acceptsValue((1,)));
        }

        {
            ListDataType t(AbstractDataProviderType::get("int"));
            assertEq("list<int>", t.getName());
            assertEq("list<int>", t.getValueType().getName());
            assertThrows("RUNTIME-TYPE-ERROR", "list.*integer", \t.acceptsValue(), 1);
            assertThrows("RUNTIME-TYPE-ERROR", "expects.*got type 'integer'", \t.acceptsValue(), (1,));
            assertThrows("RUNTIME-TYPE-ERROR", "int.*string", \t.acceptsValue(), (("a",),));
            assertThrows("RUNTIME-TYPE-ERROR", "expects.*got type 'NULL'", \t.acceptsValue(), (NULL,));
            assertEq((1,), t.acceptsValue((1,)));
        }

        {
            SoftListDataType t(StringType);
            assertEq("softlist<string>", t.getName());
            assertEq("softlist<string>", t.getValueType().getName());
            assertThrows("RUNTIME-TYPE-ERROR", "integer.*string", \t.acceptsValue(), 1);
            assertEq(("a",), t.acceptsValue(("a",)));
            assertEq(("a",), t.acceptsValue("a"));
        }

        {
            SoftListDataType t(AbstractDataProviderType::get("string"));
            assertEq("softlist<string>", t.getName());
            assertEq("softlist<string>", t.getValueType().getName());
            assertThrows("RUNTIME-TYPE-ERROR", "integer.*string", \t.acceptsValue(), 1);
            assertEq(("a",), t.acceptsValue(("a",)));
            assertEq(("a",), t.acceptsValue("a"));
        }

        {
            QoreListDataType t(new Type("softlist<int>"));
            assertEq((1,), t.acceptsValue(1));
            assertThrows("RUNTIME-TYPE-ERROR", "NULL", \t.acceptsValue(), (NULL,));
            assertEq((1,), t.acceptsValue((1,)));
        }

        {
            QoreListDataType t(new Type("softlist<string>"));
            assertEq(("a",), t.acceptsValue("a"));
            assertThrows("RUNTIME-TYPE-ERROR", "NULL", \t.acceptsValue(), (NULL,));
            assertEq(("a",), t.acceptsValue(("a",)));
        }
    }
}
