#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

%new-style
%enable-all-warnings
%require-types
%strict-args
%no-child-restrictions

# tests are made where the return value is ignored
%disable-warning return-value-ignored

%requires ../../../../qlib/QUnit.qm

%exec-class StringTest

class StringTest inherits QUnit::Test {
    constructor() : QUnit::Test("String test", "1.0") {
        addTestCase("Base64 URL test", \testBase64Url());
        addTestCase("split regex", \splitRegexTest());
        addTestCase("non scalar test", \nonScalarTest());
        addTestCase("width test", \testWidth());
        addTestCase("Basic functions test", \testBasics());
        addTestCase("Comparison test", \testComparison());
        addTestCase("Encoding test", \testEncoding());
        addTestCase("UTF-16 test", \testUtf16());
        addTestCase("ASCII x UTF-8 string equality tests", \testASCIIUTF8Equality());
        addTestCase("Base64 and hex test", \testBase64Hex());
        addTestCase("Splice test", \testSplice());
        addTestCase("Extract test", \testExtract());
        addTestCase("Join test", \testJoin());
        addTestCase("Transliteration test", \testTransliteration());
        addTestCase("Regex test", \testRegex());
        addTestCase("Chomp and trim test", \testChompAndTrim());
        addTestCase("Float strings test", \testFloat());
        addTestCase("String conversion test", \testConversions());
        addTestCase("trim", \testTrim());
        set_return_value(main());
    }

    testBase64Url() {
        string str = "<div dir=\"ltr\"><br><img src=\"test\" alt=\"?\" height=\"1\" width=\"1\"></div>";
        string base64_url = "PGRpdiBkaXI9Imx0ciI-PGJyPjxpbWcgc3JjPSJ0ZXN0IiBhbHQ9Ij8iIGhlaWdodD0iMSIgd2lkdGg9IjEiPjwvZGl2Pg";
        string base64 = "PGRpdiBkaXI9Imx0ciI+PGJyPjxpbWcgc3JjPSJ0ZXN0IiBhbHQ9Ij8iIGhlaWdodD0iMSIgd2lkdGg9IjEiPjwvZGl2Pg==";

        assertEq(base64_url, make_base64_url_string(str));
        assertEq(base64, make_base64_string(str));
        #assertEq(str, parse_base64_url_string_to_string(base64_url));
        assertEq(str, parse_base64_string_to_string(base64));
    }

    splitRegexTest() {
        string str = "gee this is a long string";
        list<string> a = str.splitRegex(" ");
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq("is", a[2]);
        assertEq("string", a[5]);

        a = str.splitRegex(" ", True);
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq("is ", a[2]);
        assertEq("string", a[5]);

        str = "field1.field2\\.with\\.dots.field3";
        a = str.splitRegex("(?<!\\\\)\\.");
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq(("field1", "field2\\.with\\.dots", "field3"), a);

        a = str.splitRegex("(?<!\\\\)\\.", True);
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq(("field1.", "field2\\.with\\.dots.", "field3"), a);

        a = str.splitRegex("xxx");
        assertTrue(a.size() == 1);
        assertTrue(a[0] == str);

        a = str.splitRegex("FIELD", RE_Caseless);
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq(("", "1.", "2\\.with\\.dots.", "3"), a);

        a = str.splitRegex("FIELD", RE_Caseless, True);
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq(("field", "1.field", "2\\.with\\.dots.field", "3"), a);

        str = str.upr();
        a = str.splitRegex("field", RE_Caseless, True);
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq(("FIELD", "1.FIELD", "2\\.WITH\\.DOTS.FIELD", "3"), a);
    }

    nonScalarTest() {
        {
            Program p(PO_NEW_STYLE);
            *hash<ExceptionInfo> warn = p.parse("string str = 'a'; hash<auto> v = {}; auto x = str + v; remove x;", "",
                -1);
            assertEq("INVALID-OPERATION", warn.err);
        }
        {
            Program p(PO_NEW_STYLE | PO_STRICT_TYPES);
            assertThrows("PARSE-TYPE-ERROR", \p.parse(), ("string str = 'a'; hash<auto> v = {}; auto x = str + v; "
                "remove x;", ""));
        };
        {
            Program p(PO_NEW_STYLE);
            *hash<ExceptionInfo> warn = p.parse("string str = 'a'; str += {};", "", -1);
            assertEq("INVALID-OPERATION", warn.err);
        }
        {
            Program p(PO_NEW_STYLE | PO_STRICT_TYPES);
            assertThrows("PARSE-TYPE-ERROR", \p.parse(), ("string str = 'a'; str += {};", ""));
        };
        {
            Program p(PO_NEW_STYLE);
            *hash<ExceptionInfo> warn = p.parse("auto sub test() { string str = 'a'; *list<auto> v = (); return str + v; }", "",
                -1);
            assertNothing(warn.err);
            assertEq(("a",), p.callFunction("test"));
        }
    }

    testWidth() {
        # 6 characters, 12 positions, 18 bytes
        string str = "ステップの例";
        assertEq(6, str.length());
        assertEq(12, str.width());
        assertEq(18, str.size());
        assertEq(str + " ", sprintf("%-13w", str));
        assertEq(str + " ", vsprintf("%-13w", str));
        assertEq("ステ.", f_sprintf("%-5w", str));
        assertEq("ステ.", f_vsprintf("%-5w", str));
    }

    testBasics() {
        string str = "Hi there, you there, pal";
        string big = "GEE WHIZ";
        assertEq(strlen(str), 24, "strlen()");
        assertEq(str.strlen(), 24, "<string>::strlen()");
        assertEq(str.size(), 24, "<string::size()");
        assertEq(toupper(str), "HI THERE, YOU THERE, PAL", "toupper()");
        assertEq(str.upr(), "HI THERE, YOU THERE, PAL", "<string>::upr()");
        assertEq(big.lwr(), "gee whiz", "<string>::lwr()");
        assertEq(reverse(big), "ZIHW EEG", "reverse()");

        assertEq(strmul(big, 2), "GEE WHIZGEE WHIZ", "strmul() basic");
        assertEq(strmul("%v, ", 3, 2), "%v, %v, %v", "strmul() extended");
        assertEq(strmul(123, 2), "123123", "strmul() type conversion");

        assertEq(chr(104), "h", "chr()");

        str = "gee this is a long string";
        list<string> a = split(" ", str);
        assertEq(True, a.complexType());
        assertEq("list<string>", a.fullType());
        assertEq(a[2], "is", "first string split()");
        assertEq(a[5], "string", "second string split()");

        list<binary> bl = split(binary(" "), binary(str));
        assertEq(bl[2], binary("is"), "first binary split()");
        assertEq(bl[5], binary("string"), "second binary split()");
    }

    testComparison() {
        assertTrue("a" < "b");
        assertTrue("a" <= "b");
        assertTrue("b" > "a");
        assertTrue("b" >= "a");
        assertTrue("ab" > "a");
        assertTrue("a" < "ab");
    }

    testEncoding() {
        # set up a string with UTF-8 multi-byte characters
        string str = "Über die Wolken läßt sich die Höhe begrüßen";
        assertEq(strlen(str), 49, "UTF-8 strlen()");
        assertEq(str.strlen(), 49, "UTF-8 <string>::strlen()");
        assertEq(length(str), 43, "UTF-8 length()");
        assertEq(str.length(), 43, "UTF-8 <string>::length()");
        assertEq(substr(str, 30), "Höhe begrüßen", "first UTF-8 substr()");
        assertEq(substr(str, -8), "begrüßen", "second UTF-8 substr()");
        assertEq(substr(str, 0, -39), "Über", "third UTF-8 substr()");
        assertEq(index(str, "läßt"), 16, "first UTF-8 index()");
        assertEq(index(str, "läßt", 1), 16, "second UTF-8 index()");
        assertEq(rindex(str, "ß"), 40, "first UTF-8 rindex()");
        assertEq(rindex(str, "ß", 25), 18, "second UTF-8 rindex()");
        assertEq(bindex(str, "läßt"), 17, "first UTF-8 bindex()");
        assertEq(bindex(str, "läßt", 1), 17, "second UTF-8 bindex()");
        assertEq(brindex(str, "ß"), 45, "first UTF-8 brindex()");
        assertEq(brindex(str, "ß", 25), 20, "second UTF-8 brindex()");
        assertEq(reverse(str), "neßürgeb ehöH eid hcis tßäl nekloW eid rebÜ", "UTF-8 reverse()");
        assertEq(index(str, "==="), -1, "negative index()");
        assertEq(rindex(str, "==="), -1, "negative rindex()");
        assertEq(bindex(str, "==="), -1, "negative bindex()");

        assertEq(str[31], "ö", "first UTF-8 string index dereference");
        assertEq(str[39], "ü", "second UTF-8 string index dereference");

        # save string
        string ostr = str;
        # convert the string to single-byte ISO-8859-1 characters and retest
        str = convert_encoding(str, "ISO-8859-1");
        assertEq(str != ostr, False, "string != operator with same text with different encodings");
        assertEq(strlen(str), 43, "ISO-8859-1 strlen()");
        assertEq(str.strlen(), 43, "ISO-8859-1 <string>::strlen()");
        assertEq(length(str), 43, "ISO-8859-1 length()");
        assertEq(str.length(), 43, "ISO-8859-1 <string>::length()");
        assertEq(substr(str, 30), convert_encoding("Höhe begrüßen", "ISO-8859-1"), "first ISO-8859-1 substr()");
        assertEq(substr(str, -8), convert_encoding("begrüßen", "ISO-8859-1"), "second ISO-8859-1 substr()");
        assertEq(substr(str, 0, -39), convert_encoding("Über", "ISO-8859-1"), "third ISO-8859-1 substr()");
        assertEq(index(str, convert_encoding("läßt", "ISO-8859-1")), 16, "first ISO-8859-1 index()");
        assertEq(index(str, convert_encoding("läßt", "ISO-8859-1"), 1), 16, "second ISO-8859-1 index()");
        assertEq(rindex(str, convert_encoding("ß", "ISO-8859-1")), 40, "first ISO-8859-1 rindex()");
        assertEq(rindex(str, convert_encoding("ß", "ISO-8859-1"), 25), 18, "second ISO-8859-1 rindex()");
        assertEq(ord(str, 1), 98, "ord()");

        str = "äüößÄÖÜ";
        # test length() with UTF-8 multi-byte characters
        assertEq(length(str), 7, "length() with UTF-8 multi-byte characters");
        assertEq(strlen(str), 14, "strlen() with UTF-8 multi-byte characters");
        # test charset encoding conversions
        string nstr = convert_encoding(str, "ISO-8859-1");
        assertEq(length(nstr), 7, "length() with ISO-8859-1 special characters");
        assertEq(strlen(nstr), 7, "strlen() with ISO-8859-1 special characters");
        assertEq(str, convert_encoding(nstr, "UTF-8"), "convert_encoding()");

        str = "příliš žluťoučký kůň úpěl ďábelské ódy";
        assertEq("prilis zlutoucky kun upel dabelske ody", str.unaccent(), "<string>::unaccent()");
        string ustr = str.upr();
        assertEq("PŘÍLIŠ ŽLUŤOUČKÝ KŮŇ ÚPĚL ĎÁBELSKÉ ÓDY", ustr, "<string>::upr()");
        assertEq("příliš žluťoučký kůň úpěl ďábelské ódy", ustr.lwr(), "<string>::lwr()");

        assertThrows("ENCODING-CONVERSION-ERROR", \convert_encoding(), ("ß", "US-ASCII"));
        assertThrows("ENCODING-CONVERSION-ERROR", \convert_encoding(), ("汉字", "ISO-8859-1"));
    }

    testUtf16() {
        string a = convert_encoding("xbcd", "UTF16BE");
        string b = convert_encoding("aaaa", "UTF16BE");
        assertTrue(a != b);
        assertFalse(a != a);
        assertFalse(a == b);
        assertFalse(a === b);
        assertTrue(a == a);
        assertTrue(a === a);
        assertTrue(a > b);
        assertTrue(a >= b);
        assertFalse(a > a);
        assertTrue(a >= a);
        assertTrue(b < a);
        assertTrue(b <= a);
        assertFalse(b < b);
        assertTrue(b <= b);
        assertEq(1, a <=> b);
        assertEq(0, a <=> a);
        assertEq(-1, b <=> a);
        assertTrue(b.equalPartial(b));
        assertFalse(b.equalPartial(a));
        assertFalse(b.equalPartial(b+b));
        assertTrue(b.equalPartial(b.substr(2)));
        assertTrue(b.equalPartial(b.substr(2, 1)));
        assertTrue(b.equalPartial(b.substr(2, -1)));
        assertTrue(b.equalPartial(substr(b, 2)));
        assertTrue(b.equalPartial(substr(b, 2, 1)));
        assertTrue(b.equalPartial(substr(b, 2, -1)));

        string c = convert_encoding("abcd", "UTF16LE");
        string d = convert_encoding("aaaa", "UTF16LE");
        assertTrue(c != d);
        assertFalse(c != c);
        assertFalse(c == d);
        assertFalse(c === d);
        assertTrue(c == c);
        assertTrue(c === c);
        assertTrue(c > d);
        assertTrue(c >= d);
        assertFalse(c > c);
        assertTrue(c >= c);
        assertTrue(d < c);
        assertTrue(d <= c);
        assertFalse(d < d);
        assertTrue(d <= d);
        assertEq(1, c <=> d);
        assertEq(0, c <=> c);
        assertEq(-1, d <=> c);
        assertTrue(d.equalPartial(d));
        assertFalse(d.equalPartial(c));
        assertFalse(d.equalPartial(d+d));
        assertTrue(d.equalPartial(d.substr(2)));
        assertTrue(d.equalPartial(d.substr(2, 1)));
        assertTrue(d.equalPartial(d.substr(2, -1)));
        assertTrue(d.equalPartial(substr(d, 2)));
        assertTrue(d.equalPartial(substr(d, 2, 1)));
        assertTrue(d.equalPartial(substr(d, 2, -1)));

        a = convert_encoding("\nabcd\n", "UTF16BE");
        assertEq("abcd", trim(a));
        a = convert_encoding("\nabcd\n", "UTF16BE");
        assertEq("abcd", trim(a, "12\n3"));
        a = convert_encoding("\nabcd\n", "UTF16BE");
        assertEq("\nabcd", rtrim(a));
        a = convert_encoding("\nabcd\n", "UTF16BE");
        assertEq("\nabcd", rtrim(a, "12\n3"));
        a = convert_encoding("\nabcd\n", "UTF16BE");
        assertEq("abcd\n", ltrim(a));
        a = convert_encoding("\nabcd\n", "UTF16BE");
        assertEq("abcd\n", ltrim(a, "12\n3"));
        a = convert_encoding("\nabcd\n", "UTF16BE");
        assertEq("abcd", trim a);

        a = convert_encoding("\r\nabcd\r\n", "UTF16BE");
        assertEq("abcd", trim(a));
        a = convert_encoding("\r\nabcd\r\n", "UTF16BE");
        assertEq("abcd", trim(a, "12\r\n3"));
        a = convert_encoding("\r\nabcd\r\n", "UTF16BE");
        assertEq("\r\nabcd", rtrim(a));
        a = convert_encoding("\r\nabcd\r\n", "UTF16BE");
        assertEq("\r\nabcd", rtrim(a, "12\r\n3"));
        a = convert_encoding("\r\nabcd\r\n", "UTF16BE");
        assertEq("abcd\r\n", ltrim(a));
        a = convert_encoding("\r\nabcd\r\n", "UTF16BE");
        assertEq("abcd\r\n", ltrim(a, "12\r\n3"));
        a = convert_encoding("\r\nabcd\r\n", "UTF16BE");
        assertEq("abcd", trim a);

        a = convert_encoding("\rabcd\r", "UTF16BE");
        assertEq("abcd", trim(a));
        a = convert_encoding("\rabcd\r", "UTF16BE");
        assertEq("abcd", trim(a, "12\r3"));
        a = convert_encoding("\rabcd\r", "UTF16BE");
        assertEq("\rabcd", rtrim(a));
        a = convert_encoding("\rabcd\r", "UTF16BE");
        assertEq("\rabcd", rtrim(a, "12\r3"));
        a = convert_encoding("\rabcd\r", "UTF16BE");
        assertEq("abcd\r", ltrim(a));
        a = convert_encoding("\rabcd\r", "UTF16BE");
        assertEq("abcd\r", ltrim(a, "12\r3"));
        a = convert_encoding("\rabcd\r", "UTF16BE");
        assertEq("abcd", trim a);

        a = convert_encoding("\nabcd\n", "UTF16LE");
        assertEq("abcd", trim(a));
        a = convert_encoding("\nabcd\n", "UTF16LE");
        assertEq("abcd", trim(a, "12\n3"));
        a = convert_encoding("\nabcd\n", "UTF16LE");
        assertEq("\nabcd", rtrim(a));
        a = convert_encoding("\nabcd\n", "UTF16LE");
        assertEq("\nabcd", rtrim(a, "12\n3"));
        a = convert_encoding("\nabcd\n", "UTF16LE");
        assertEq("abcd\n", ltrim(a));
        a = convert_encoding("\nabcd\n", "UTF16LE");
        assertEq("abcd\n", ltrim(a, "12\n3"));
        a = convert_encoding("\nabcd\n", "UTF16LE");
        assertEq("abcd", trim a);

        a = convert_encoding("\r\nabcd\r\n", "UTF16LE");
        assertEq("abcd", trim(a));
        a = convert_encoding("\r\nabcd\r\n", "UTF16LE");
        assertEq("abcd", trim(a, "12\r\n3"));
        a = convert_encoding("\r\nabcd\r\n", "UTF16LE");
        assertEq("\r\nabcd", rtrim(a));
        a = convert_encoding("\r\nabcd\r\n", "UTF16LE");
        assertEq("\r\nabcd", rtrim(a, "12\r\n3"));
        a = convert_encoding("\r\nabcd\r\n", "UTF16LE");
        assertEq("abcd\r\n", ltrim(a));
        a = convert_encoding("\r\nabcd\r\n", "UTF16LE");
        assertEq("abcd\r\n", ltrim(a, "12\r\n3"));
        a = convert_encoding("\r\nabcd\r\n", "UTF16LE");
        assertEq("abcd", trim a);

        a = convert_encoding("\rabcd\r", "UTF16LE");
        assertEq("abcd", trim(a));
        a = convert_encoding("\rabcd\r", "UTF16LE");
        assertEq("abcd", trim(a, "12\r3"));
        a = convert_encoding("\rabcd\r", "UTF16LE");
        assertEq("\rabcd", rtrim(a));
        a = convert_encoding("\rabcd\r", "UTF16LE");
        assertEq("\rabcd", rtrim(a, "12\r3"));
        a = convert_encoding("\rabcd\r", "UTF16LE");
        assertEq("abcd\r", ltrim(a));
        a = convert_encoding("\rabcd\r", "UTF16LE");
        assertEq("abcd\r", ltrim(a, "12\r3"));
        a = convert_encoding("\rabcd\r", "UTF16LE");
        assertEq("abcd", trim a);

        assertEq("ABCD", toupper(a));
        assertEq("ABCD", a.upr());
        assertEq("abcd", tolower(toupper(a)));
        assertEq("abcd", a.upr().lwr());
        assertEq(("a", "cd"), split(convert_encoding("b", "UTF16LE"), a));
        assertEq(("a", "cd"), a.split(convert_encoding("b", "UTF16LE")));
        assertEq(("a", "cd"), split(convert_encoding("b", "UTF16BE"), a));
        assertEq(("a", "cd"), a.split(convert_encoding("b", "UTF16BE")));

        string lstr = convert_encoding("a,b,c", "UTF16LE");
        assertEq(("a", "b", "c"), split(convert_encoding(",", "UTF16LE"), lstr));
        assertEq(("a", "b", "c"), lstr.split(convert_encoding(",", "UTF16LE")));
        lstr = convert_encoding("a,'b,c,d',e", "UTF16LE");
        assertEq(("a", "b,c,d", "e"), split(convert_encoding(",", "UTF16LE"), lstr, convert_encoding("'", "UTF16LE")));
        assertEq(("a", "b,c,d", "e"), lstr.split(convert_encoding(",", "UTF16LE"), convert_encoding("'", "UTF16LE")));

        lstr = convert_encoding("a,b,c", "UTF16BE");
        assertEq(("a", "b", "c"), split(convert_encoding(",", "UTF16BE"), lstr));
        assertEq(("a", "b", "c"), lstr.split(convert_encoding(",", "UTF16BE")));
        lstr = convert_encoding("a,'b,c,d',e", "UTF16BE");
        assertEq(("a", "b,c,d", "e"), split(convert_encoding(",", "UTF16BE"), lstr, convert_encoding("'", "UTF16BE")));
        assertEq(("a", "b,c,d", "e"), lstr.split(convert_encoding(",", "UTF16BE"), convert_encoding("'", "UTF16BE")));

        # test strings with double quotes and escaped quotes as part of the field
        lstr = "a,'',b,'c','''d''',e,'f''1'''";
        assertEq(("a", "", "b", "c","'d'","e","f'1'"), lstr.split(",", "'"));
        lstr = "a,b'c','\\'d','e''1',f";
        assertEq(("a", "b'c'","'d","e'1","f"), lstr.split(",", "'"));
        assertThrows("SPLIT-ERROR", \split(), (",", "a,'asdsad", "'"));
        assertThrows("SPLIT-ERROR", \split(), (",", "a,''ad", "'"));
        assertThrows("SPLIT-ERROR", \split(), (",", "a,'ad'x,b", "'"));

        # test currently unsupported operations
        assertThrows("UNSUPPORTED-ENCODING", \html_encode(), a);
        assertThrows("UNSUPPORTED-ENCODING", \html_decode(), a);
        assertThrows("UNSUPPORTED-ENCODING", \encode_url(), a);
        assertThrows("UNSUPPORTED-ENCODING", \encode_uri_request(), a);
        assertThrows("UNSUPPORTED-ENCODING", sub (string str) { str.equalPartialPath(str); }, a);
        assertThrows("UNSUPPORTED-ENCODING", sub (string str) { str.toInt(16); }, a);
    }

    testASCIIUTF8Equality() {
        string a = convert_encoding("ěščřžýáíé", "UTF-8");
        string b = convert_encoding("escrzyaie", "ASCII");
        assertFalse(a == b);
        assertTrue(a != b);
        assertFalse(b == a);
        assertTrue(b != a);
        assertFalse(a.equalPartial(b));
        assertFalse(b.equalPartial(a));
        assertFalse(a.equalPartialPath(b));
        assertFalse(b.equalPartialPath(a));
    }

    testBase64Hex() {
        # assign binary object
        binary x = <0abf83e8ca72d32c>;
        string b64 = make_base64_string(x);
        assertEq(x, parse_base64_string(b64), "first base64");
        assertEq("aGVsbG8=", make_base64_string("hello"), "make_base64_string()");
        string b64url = make_base64_url_string(x);
        assertEq(x, parse_base64_url_string(b64url), "first base64-url");
        assertEq("aGVsbG8", make_base64_url_string("hello"), "first make_base64_url_string()");
        b64url = x.toBase64Url();
        assertEq(x, parse_base64_url_string(b64url), "first <string>::toBase64Url()");
        assertEq("aGVsbG8", make_base64_url_string("hello"), "second make_base64_url_string()");

        binary b = get_random_bytes(1024);
        b64 = make_base64_string(b);
        assertEq(b, parse_base64_string(b64));
        b64 = make_base64_url_string(b);
        assertEq(b, parse_base64_url_string(b64));

        string hex = make_hex_string(x);
        assertEq(x, parse_hex_string(hex), "first hex");
        assertEq("", parse_base64_string_to_string(""));
    }

    testSplice() {
        # UTF-8 string splice operator tests
        string str = "äbcdéf";
        splice str, 5;
        assertEq(str, "äbcdé", "first UTF-8 string splice");
        splice str, 3, 1;
        assertEq(str, "äbcé", "second UTF-8 string splice");
        splice str, 1, 2, "GHI";
        assertEq(str, "äGHIé", "third UTF-8 string splice");
        splice str, 0, 4, "jkl";
        assertEq(str, "jklé", "fourth UTF-8 string splice");
        splice str, 0, 1;
        assertEq(str, "klé", "fifth UTF-8 string splice");
        splice str, 5, 2, "MNO";
        assertEq(str, "kléMNO", "sixth UTF-8 string splice");
        splice str, -4, 2, "p";
        assertEq(str, "klpNO", "seventh UTF-8 string splice");
        splice str, -4, -2, "QRS";
        assertEq(str, "kQRSNO", "eighth UTF-8 string splice");

        # single-byte string splice operator tests
        str = convert_encoding("äbcdéf", "ISO-8859-1");
        splice str, 5;
        assertTrue(str == "äbcdé", "first ISO-8859-1 string splice");
        splice str, 3, 1;
        assertTrue(str == "äbcé", "second ISO-8859-1 string splice");
        splice str, 1, 2, "GHI";
        assertTrue(str == "äGHIé", "third ISO-8859-1 string splice");
        splice str, 0, 4, "jkl";
        assertTrue(str == "jklé", "fouth ISO-8859-1 string splice");
        splice str, 0, 1;
        assertTrue(str == "klé", "fifth ISO-8859-1 string splice");
        splice str, 5, 2, "MNO";
        assertTrue(str == "kléMNO", "sixth ISO-8859-1 string splice");
        splice str, -4, 2, "p";
        assertTrue(str == "klpNO", "seventh ISO-8859-1 string splice");
        splice str, -4, -2, "QRS";
        assertTrue(str == "kQRSNO", "eighth ISO-8859-1 string splice");

        # UTF-8 string splice function tests
        str = "äbcdéf";
        str = splice str, 5;
        assertEq(str, "äbcdé", "first UTF-8 string splice");
        str = splice str, 3, 1;
        assertEq(str, "äbcé", "second UTF-8 string splice");
        str = splice str, 1, 2, "GHI";
        assertEq(str, "äGHIé", "third UTF-8 string splice");
        str = splice str, 0, 4, "jkl";
        assertEq(str, "jklé", "fourth UTF-8 string splice");
        str = splice str, 0, 1;
        assertEq(str, "klé", "fifth UTF-8 string splice");
        str = splice str, 5, 2, "MNO";
        assertEq(str, "kléMNO", "sixth UTF-8 string splice");
        str = splice str, -4, 2, "p";
        assertEq(str, "klpNO", "seventh UTF-8 string splice");
        str = splice str, -4, -2, "QRS";
        assertEq(str, "kQRSNO", "eighth UTF-8 string splice");

        # single-byte string splice function tests
        str = convert_encoding("äbcdéf", "ISO-8859-1");
        str = splice str, 5;
        assertTrue(str == "äbcdé", "first ISO-8859-1 string splice");
        str = splice str, 3, 1;
        assertTrue(str == "äbcé", "second ISO-8859-1 string splice");
        str = splice str, 1, 2, "GHI";
        assertTrue(str == "äGHIé", "third ISO-8859-1 string splice");
        str = splice str, 0, 4, "jkl";
        assertTrue(str == "jklé", "fouth ISO-8859-1 string splice");
        str = splice str, 0, 1;
        assertTrue(str == "klé", "fifth ISO-8859-1 string splice");
        str = splice str, 5, 2, "MNO";
        assertTrue(str == "kléMNO", "sixth ISO-8859-1 string splice");
        str = splice str, -4, 2, "p";
        assertTrue(str == "klpNO", "seventh ISO-8859-1 string splice");
        str = splice str, -4, -2, "QRS";
        assertTrue(str == "kQRSNO", "eighth ISO-8859-1 string splice");
    }

    testExtract() {
        # UTF-8 string extract tests
        string str = "äbcdéf";
        assertEq((extract str, 4), "éf", "first UTF-8 string extract");
        assertEq((extract str, 1, 2), "bc", "second UTF-8 string extract");
        assertEq((extract str, 1, 1, "bcdef"), "d", "third UTF-8 string extract");
        assertEq(str, "äbcdef", "final UTF-8 string extract");

        str = convert_encoding("äbcdéf", "ISO-8859-1");
        string val = extract str, 4;
        assertTrue(val == "éf", "first UTF-8 string extract");
        val = extract str, 1, 2;
        assertTrue(val == "bc", "second UTF-8 string extract");
        val = extract str, 1, 1, "bcdef";
        assertTrue(val == "d", "third UTF-8 string extract");
        assertTrue(str == "äbcdef", "final UTF-8 string extract");
    }

    testJoin() {
        string str = join(":", "login","passwd",1,"gid","gcos","home","shell");
        assertEq(str, "login:passwd:1:gid:gcos:home:shell", "first join");
        list l = ("login","passwd","uid","gid","gcos","home","shell");
        str = join(":", l);
        assertEq(str, "login:passwd:uid:gid:gcos:home:shell", "second join");
    }

    testTransliteration() {
        string str = "Hello There";
        assertEq(str =~ tr/def/123/, "H2llo Th2r2", "first transliteration");
        assertEq(str =~ tr/a-z/0-9/, "H2999 T7292", "first range transliteration");
        str = "Hello There";
        assertEq(str =~ tr/a-z/A-Z/, "HELLO THERE", "second range transliteration");
        assertEq(str =~ tr/A-Z/a-z/, "hello there", "third range transliteration");
    }

    testRegex() {
        # regex subpattern extraction operator tests
        assertEq("xmlns:wsdl" =~ x/(\w+):(\w+)/, ("xmlns", "wsdl"), "regex subpattern extraction");
        assertEq("xmlns-wsdl" =~ x/(\w+):(\w+)/, NOTHING, "negative regex subpattern extraction");

        # unicode match tests
        assertNothing("ステップの例:1.0" =~ x/(\w+):([[:alnum:]\._]+)/);
        assertEq(("ステップの例", "1.0"), ("ステップの例:1.0" =~ x/(\w+):([[:alnum:]\._]+)/u));
        assertNothing(regex_extract("ステップの例:1.0", "(\\w+):([[:alnum:]\\._]+)"));
        assertEq(("ステップの例", "1.0"), regex_extract("ステップの例:1.0", "(\\w+):([[:alnum:]\\._]+)", RE_Unicode));

        assertEq(regex_extract("xmlns:wsdl", "(\\w+):(\\w+)"), ("xmlns", "wsdl"), "regex_extract()");
        {
            *list<*string> l = ("xmlns:wsdl" =~ x/(\w+):(\w+)/);
            assertEq(True, l.complexType());
            assertEq("list<*string>", l.fullType());
            l = regex_extract("xmlns:wsdl", "(\\w+):(\\w+)");
            assertEq(True, l.complexType());
            assertEq("list<*string>", l.fullType());
        }

        # regex operator tests
        assertTrue("hello" =~ /^hel*/, "regular expression positive match");
        assertFalse("hello" =~ m/123*/, "regular expression negative match");
        assertTrue("hello" =~ m/^HEL*/i, "regular expression case-insensitive positive match");
        assertFalse("hello" =~ /^HEL*/, "regular expression case-insensitive negative match");
        assertTrue("test\nx" =~ /test.x/s, "regular expression newline positive match");
        assertFalse("test\nx" =~ /test.x/, "regular expression newline negative match");
        assertTrue("hello" =~ /^  hel* #comment/x, "extended regular expression positive match");
        assertFalse("hello" =~ /^  hel* #comment/, "extended regular expression negative match");
        assertTrue("test\nx123" =~ /^x123/m, "multiline regular expression positive match");
        assertFalse("test\nx123" =~ /^x123/, "multiline regular expression negative match");
        # NOTE that escaping UTF-8 characters (\ä) doesn't work on PPC for some reason
        assertTrue("testäöüß" =~ /äöüß/, "regular expression UTF-8 match");
        assertFalse("testäöüß" =~ /aouB/, "regular expression UTF-8 negative match");
        assertFalse("hello" !~ /hel*/, "negative regular expression negative match");
        assertTrue("hello" !~ /123*/, "negative regular expression positive match");

        # regex substitution operator tests
        list l = ( "hello bar hi bar", "bar hello bar hi bar", "hello bar hi" );
        assertEq(l[0] =~ s/bar/foo/, "hello foo hi bar", "first non-global regular expression substitution");
        assertEq(l[1] =~ s/bar/foo/, "foo hello bar hi bar", "second non-global regular expression substitution");
        assertEq(l[2] =~ s/BAR/foo/i, "hello foo hi", "case-insensitive non-global regular expression substitution");
        l = ( "hello bar hi bar", "bar hello bar hi bar", "hello bar hi" );
        assertEq(l[0] =~ s/bar/foo/g, "hello foo hi foo", "first global regular expression substitution");
        assertEq(l[1] =~ s/bar/foo/g, "foo hello foo hi foo", "second global regular expression substitution");
        assertEq(l[2] =~ s/BAR/foo/ig, "hello foo hi", "case-insensitive global regular expression substitution");

        string astr= "abc def";
        astr =~ s/(\w+) +(\w+)/$2, $1/;
        assertEq(astr, "def, abc", "regular expression subpattern substitution");

        # regex() tests
        assertTrue(regex("hello", "^hel*"), "regex() positive match");
        assertFalse(regex("hello", "123*"), "regex() negative match");
        assertTrue(regex("hello", "^HEL*", RE_Caseless), "regex() case-insensitive positive match");
        assertFalse(regex("hello", "^HEL*"), "regex() case-insensitive negative match");
        assertTrue(regex("test\nx", "test.x", RE_DotAll), "regex() newline positive match");
        assertFalse(regex("test\nx", "test.x/"), "regex() newline negative match");
        assertTrue(regex("hello", "^  hel* #comment", RE_Extended), "regex() extended positive match");
        assertFalse(regex("hello", "^  hel* #comment"), "regex() extended negative match");
        assertTrue(regex("test\nx123", "^x123", RE_MultiLine), "regex() multiline positive match");
        assertFalse(regex("test\nx123", "^x123/"), "regex() multiline negative match");
        assertTrue(regex("testäöüß", "\äöüß"), "regex() UTF-8 match");
        assertFalse(regex("testäöüß", "aouB"), "regex() UTF-8 negative match");

        # regex_subst() tests
        l = ( "hello bar hi bar", "bar hello bar hi bar", "hello bar hi" );
        assertEq(regex_subst(l[0], "bar", "foo"), "hello foo hi bar", "first non-global regex_subst()");
        assertEq(regex_subst(l[1], "bar", "foo"), "foo hello bar hi bar", "second non-global regex_subst()");
        assertEq(regex_subst(l[2], "BAR", "foo", RE_Caseless), "hello foo hi", "case-insensitive non-global regex_subst()");
        l = ( "hello bar hi bar", "bar hello bar hi bar", "hello bar hi" );
        assertEq(regex_subst(l[0], "bar", "foo", RE_Global), "hello foo hi foo", "first global regex_subst()");
        assertEq(regex_subst(l[1], "bar", "foo", RE_Global), "foo hello foo hi foo", "second global regex_subst()");
        assertEq(regex_subst(l[2], "BAR", "foo", RE_Global|RE_Caseless), "hello foo hi", "case-insensitive global regex_subst()");

        astr = "abc def";
        # note that the escape characters have to be escaped in the following pattern string
        assertEq(regex_subst(astr, "(\\w+) +(\\w+)", "$2, $1"), "def, abc", "first subpattern regex_subst()");
        # here we use single-quotes, so the escape characters do not have to be escaped
        assertEq(regex_subst(astr, '(\w+) +(\w+)', "$2, $1"), "def, abc", "second subpattern regex_subst()");
    }

    testChompAndTrim() {
        # chomp tests
        string str = "hello\n";
        chomp str;
        assertEq(str, "hello", "first string chomp");
        str += "\r\n";
        chomp str;
        assertEq(str, "hello", "second string chomp");
        list l = ( 1, "hello\n", 3.0, True, "test\r\n" );
        chomp l;
        assertEq(l, ( 1, "hello", 3.0, True, "test" ), "list chomp");

        hash h = ( "key1" : "hello\n", "key2" : 2045, "key3": "test\r\n", "key4" : 302.223 );
        chomp h;
        assertEq(h, ( "key1" : "hello", "key2" : 2045, "key3": "test", "key4" : 302.223 ), "hash chomp");
        str = "hello\n";
        chomp(\str);
        assertEq(str, "hello", "string reference chomp()");
        str = "  \t\n  hello  \n   \r \t \0 ";
        trim str;
        assertEq("hello", str, "trim string operator test");
        str = "  \t\n  hello  \n   \r \t \0 ";
        trim(\str);
        assertEq("hello", str, "trim string reference test");

        str = "  \t\n  hello  \n   \r \t \0 ";
        assertEq("hello", trim(str), "trim string function test");
        assertEq("  \t\n  hello", rtrim(str), "rtrim string function test");
        assertEq("hello  \n   \r \t \0 ", ltrim(str), "ltrim string function test");

        str = "  \t\n  hello  \n   \r \t \0 ";
        rtrim(\str);
        assertEq("  \t\n  hello", str, "rtrim string reference test");
        str = "  \t\n  hello  \n   \r \t \0 ";
        ltrim(\str);
        assertEq("hello  \n   \r \t \0 ", str, "ltrim string reference test");

        l = ( 1, "   \r \t hello  \n  \r \v \t", 3.0, True, "    test\r\n  " );
        trim l;
        assertEq((1, "hello", 3.0, True, "test"), l, "list trim");

        h = {"key1" : "    hello\n \r  ", "key2" : 2045, "key3": "     test\r   \n \t\v   ", "key4" : 302.223};
        trim h;
        assertEq({"key1" : "hello", "key2" : 2045, "key3": "test", "key4" : 302.223}, h, "hash trim");

        softstring tstr = 1;
        tstr += "\n";
        chomp(\tstr);
        assertEq("1", tstr);
    }

    testFloat() {
        # make sure strings containing floating-point numbers between -1.0 and 1.0 exclusive return True when evaluated in a boolean context
        assertTrue(boolean("0.1"), "first string fp boolean");
        assertTrue(boolean("-0.1"), "second string fp boolean");

        # regression tests for floating-point formatting bugs
        assertEq(sprintf("%f", 1.5), "1.500000", "%f float");
        assertEq(sprintf("%f", 1.5n), "1.500000", "%f number");
        assertEq(sprintf("%g", 1.5), "1.5", "%f float");
        assertEq(sprintf("%g", 1.5n), "1.5", "%f number");
    }

    testConversions() {
        assertEq("", string(NULL));
    }

    testTrim() {
        string a = "\nabcd\n";
        assertEq("abcd", trim(a));
        a = "\nabcd\n";
        assertEq("abcd", trim(a, "12\n3"));
        a = "\nabcd\n";
        assertEq("\nabcd", rtrim(a));
        a = "\nabcd\n";
        assertEq("\nabcd", rtrim(a, "12\n3"));
        a = "\nabcd\n";
        assertEq("abcd\n", ltrim(a));
        a = "\nabcd\n";
        assertEq("abcd\n", ltrim(a, "12\n3"));
        a = "\nabcd\n";
        assertEq("abcd", trim a);

        a = "\r\nabcd\r\n";
        assertEq("abcd", trim(a));
        a = "\r\nabcd\r\n";
        assertEq("abcd", trim(a, "12\r\n3"));
        a = "\r\nabcd\r\n";
        assertEq("\r\nabcd", rtrim(a));
        a = "\r\nabcd\r\n";
        assertEq("\r\nabcd", rtrim(a, "12\r\n3"));
        a = "\r\nabcd\r\n";
        assertEq("abcd\r\n", ltrim(a));
        a = "\r\nabcd\r\n";
        assertEq("abcd\r\n", ltrim(a, "12\r\n3"));
        a = "\r\nabcd\r\n";
        assertEq("abcd", trim a);

        a = "\rabcd\r";
        assertEq("abcd", trim(a));
        a = "\rabcd\r";
        assertEq("abcd", trim(a, "12\r3"));
        a = "\rabcd\r";
        assertEq("\rabcd", rtrim(a));
        a = "\rabcd\r";
        assertEq("\rabcd", rtrim(a, "12\r3"));
        a = "\rabcd\r";
        assertEq("abcd\r", ltrim(a));
        a = "\rabcd\r";
        assertEq("abcd\r", ltrim(a, "12\r3"));
        a = "\rabcd\r";
        assertEq("abcd", trim a);
    }
}
