Source code
Description: source: package: org.apache.commons.text.similarity, class: LevenshteinDetailedDistance

    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017package org.apache.commons.text.similarity;
    018
    019import java.util.Arrays;
    020
    021/**
    022 * An algorithm for measuring the difference between two character sequences.
    023 *
    024 * <p>
    025 * This is the number of changes needed to change one sequence into another,
    026 * where each change is a single character modification (deletion, insertion
    027 * or substitution).
    028 * </p>
    029 *
    030 * @since 1.0
    031 */
    032public class LevenshteinDetailedDistance implements EditDistance<LevenshteinResults> {
    033
    034    /**
    035     * Singleton instance.
    036     */
    037    private static final LevenshteinDetailedDistance INSTANCE = new LevenshteinDetailedDistance();
    038
    039    /**
    040     * Finds count for each of the three [insert, delete, substitute] operations
    041     * needed. This is based on the matrix formed based on the two character
    042     * sequence.
    043     *
    044     * @param left character sequence which need to be converted from
    045     * @param right character sequence which need to be converted to
    046     * @param matrix two dimensional array containing
    047     * @param swapped tells whether the value for left character sequence and right
    048     *            character sequence were swapped to save memory
    049     * @return result object containing the count of insert, delete and substitute and total count needed
    050     */
    051    private static LevenshteinResults findDetailedResults(final CharSequence left,
    052                                                          final CharSequence right,
    053                                                          final int[][] matrix,
    054                                                          final boolean swapped) {
    055
    056        int delCount = 0;
    057        int addCount = 0;
    058        int subCount = 0;
    059
    060        int rowIndex = right.length();
    061        int columnIndex = left.length();
    062
    063        int dataAtLeft = 0;
    064        int dataAtTop = 0;
    065        int dataAtDiagonal = 0;
    066        int data = 0;
    067        boolean deleted = false;
    068        boolean added = false;
    069
    070        while (rowIndex >= 0 && columnIndex >= 0) {
    071
    072            if (columnIndex == 0) {
    073                dataAtLeft = -1;
    074            } else {
    075                dataAtLeft = matrix[rowIndex][columnIndex - 1];
    076            }
    077            if (rowIndex == 0) {
    078                dataAtTop = -1;
    079            } else {
    080                dataAtTop = matrix[rowIndex - 1][columnIndex];
    081            }
    082            if (rowIndex > 0 && columnIndex > 0) {
    083                dataAtDiagonal = matrix[rowIndex - 1][columnIndex - 1];
    084            } else {
    085                dataAtDiagonal = -1;
    086            }
    087            if (dataAtLeft == -1 && dataAtTop == -1 && dataAtDiagonal == -1) {
    088                break;
    089            }
    090            data = matrix[rowIndex][columnIndex];
    091
    092            // case in which the character at left and right are the same,
    093            // in this case none of the counters will be incremented.
    094            if (columnIndex > 0 && rowIndex > 0 && left.charAt(columnIndex - 1) == right.charAt(rowIndex - 1)) {
    095                columnIndex--;
    096                rowIndex--;
    097                continue;
    098            }
    099
    100            // handling insert and delete cases.
    101            deleted = false;
    102            added = false;
    103            if (data - 1 == dataAtLeft && data <= dataAtDiagonal && data <= dataAtTop
    104                    || dataAtDiagonal == -1 && dataAtTop == -1) { // NOPMD
    105                columnIndex--;
    106                if (swapped) {
    107                    addCount++;
    108                    added = true;
    109                } else {
    110                    delCount++;
    111                    deleted = true;
    112                }
    113            } else if (data - 1 == dataAtTop && data <= dataAtDiagonal && data <= dataAtLeft
    114                    || dataAtDiagonal == -1 && dataAtLeft == -1) { // NOPMD
    115                rowIndex--;
    116                if (swapped) {
    117                    delCount++;
    118                    deleted = true;
    119                } else {
    120                    addCount++;
    121                    added = true;
    122                }
    123            }
    124
    125            // substituted case
    126            if (!added && !deleted) {
    127                subCount++;
    128                columnIndex--;
    129                rowIndex--;
    130            }
    131        }
    132        return new LevenshteinResults(addCount + delCount + subCount, addCount, delCount, subCount);
    133    }
    134
    135    /**
    136     * Gets the default instance.
    137     *
    138     * @return The default instace
    139     */
    140    public static LevenshteinDetailedDistance getDefaultInstance() {
    141        return INSTANCE;
    142    }
    143
    144    /**
    145     * Finds the Levenshtein distance between two CharSequences if it's less than or
    146     * equal to a given threshold.
    147     *
    148     * <p>
    149     * This implementation follows from Algorithms on Strings, Trees and
    150     * Sequences by Dan Gusfield and Chas Emerick's implementation of the
    151     * Levenshtein distance algorithm from <a
    152     * href="http://www.merriampark.com/ld.htm"
    153     * >http://www.merriampark.com/ld.htm</a>
    154     * </p>
    155     *
    156     * <pre>
    157     * limitedCompare(null, *, *)             = IllegalArgumentException
    158     * limitedCompare(*, null, *)             = IllegalArgumentException
    159     * limitedCompare(*, *, -1)               = IllegalArgumentException
    160     * limitedCompare("","", 0)               = 0
    161     * limitedCompare("aaapppp", "", 8)       = 7
    162     * limitedCompare("aaapppp", "", 7)       = 7
    163     * limitedCompare("aaapppp", "", 6))      = -1
    164     * limitedCompare("elephant", "hippo", 7) = 7
    165     * limitedCompare("elephant", "hippo", 6) = -1
    166     * limitedCompare("hippo", "elephant", 7) = 7
    167     * limitedCompare("hippo", "elephant", 6) = -1
    168     * </pre>
    169     *
    170     * @param left the first CharSequence, must not be null
    171     * @param right the second CharSequence, must not be null
    172     * @param threshold the target threshold, must not be negative
    173     * @return result distance, or -1
    174     */
    175    private static LevenshteinResults limitedCompare(CharSequence left,
    176                                                     CharSequence right,
    177                                                     final int threshold) { //NOPMD
    178        if (left == null || right == null) {
    179            throw new IllegalArgumentException("CharSequences must not be null");
    180        }
    181        if (threshold < 0) {
    182            throw new IllegalArgumentException("Threshold must not be negative");
    183        }
    184
    185        /*
    186         * This implementation only computes the distance if it's less than or
    187         * equal to the threshold value, returning -1 if it's greater. The
    188         * advantage is performance: unbounded distance is O(nm), but a bound of
    189         * k allows us to reduce it to O(km) time by only computing a diagonal
    190         * stripe of width 2k + 1 of the cost table. It is also possible to use
    191         * this to compute the unbounded Levenshtein distance by starting the
    192         * threshold at 1 and doubling each time until the distance is found;
    193         * this is O(dm), where d is the distance.
    194         *
    195         * One subtlety comes from needing to ignore entries on the border of
    196         * our stripe eg. p[] = |#|#|#|* d[] = *|#|#|#| We must ignore the entry
    197         * to the left of the leftmost member We must ignore the entry above the
    198         * rightmost member
    199         *
    200         * Another subtlety comes from our stripe running off the matrix if the
    201         * strings aren't of the same size. Since string s is always swapped to
    202         * be the shorter of the two, the stripe will always run off to the
    203         * upper right instead of the lower left of the matrix.
    204         *
    205         * As a concrete example, suppose s is of length 5, t is of length 7,
    206         * and our threshold is 1. In this case we're going to walk a stripe of
    207         * length 3. The matrix would look like so:
    208         *
    209         * <pre>
    210         *    1 2 3 4 5
    211         * 1 |#|#| | | |
    212         * 2 |#|#|#| | |
    213         * 3 | |#|#|#| |
    214         * 4 | | |#|#|#|
    215         * 5 | | | |#|#|
    216         * 6 | | | | |#|
    217         * 7 | | | | | |
    218         * </pre>
    219         *
    220         * Note how the stripe leads off the table as there is no possible way
    221         * to turn a string of length 5 into one of length 7 in edit distance of
    222         * 1.
    223         *
    224         * Additionally, this implementation decreases memory usage by using two
    225         * single-dimensional arrays and swapping them back and forth instead of
    226         * allocating an entire n by m matrix. This requires a few minor
    227         * changes, such as immediately returning when it's detected that the
    228         * stripe has run off the matrix and initially filling the arrays with
    229         * large values so that entries we don't compute are ignored.
    230         *
    231         * See Algorithms on Strings, Trees and Sequences by Dan Gusfield for
    232         * some discussion.
    233         */
    234
    235        int n = left.length(); // length of left
    236        int m = right.length(); // length of right
    237
    238        // if one string is empty, the edit distance is necessarily the length of the other
    239        if (n == 0) {
    240            return m <= threshold ? new LevenshteinResults(m, m, 0, 0) : new LevenshteinResults(-1, 0, 0, 0);
    241        }
    242        if (m == 0) {
    243            return n <= threshold ? new LevenshteinResults(n, 0, n, 0) : new LevenshteinResults(-1, 0, 0, 0);
    244        }
    245
    246        boolean swapped = false;
    247        if (n > m) {
    248            // swap the two strings to consume less memory
    249            final CharSequence tmp = left;
    250            left = right;
    251            right = tmp;
    252            n = m;
    253            m = right.length();
    254            swapped = true;
    255        }
    256
    257        int[] p = new int[n + 1]; // 'previous' cost array, horizontally
    258        int[] d = new int[n + 1]; // cost array, horizontally
    259        int[] tempD; // placeholder to assist in swapping p and d
    260        final int[][] matrix = new int[m + 1][n + 1];
    261
    262        //filling the first row and first column values in the matrix
    263        for (int index = 0; index <= n; index++) {
    264            matrix[0][index] = index;
    265        }
    266        for (int index = 0; index <= m; index++) {
    267            matrix[index][0] = index;
    268        }
    269
    270        // fill in starting table values
    271        final int boundary = Math.min(n, threshold) + 1;
    272        for (int i = 0; i < boundary; i++) {
    273            p[i] = i;
    274        }
    275        // these fills ensure that the value above the rightmost entry of our
    276        // stripe will be ignored in following loop iterations
    277        Arrays.fill(p, boundary, p.length, Integer.MAX_VALUE);
    278        Arrays.fill(d, Integer.MAX_VALUE);
    279
    280        // iterates through t
    281        for (int j = 1; j <= m; j++) {
    282            final char rightJ = right.charAt(j - 1); // jth character of right
    283            d[0] = j;
    284
    285            // compute stripe indices, constrain to array size
    286            final int min = Math.max(1, j - threshold);
    287            final int max = j > Integer.MAX_VALUE - threshold ? n : Math.min(
    288                    n, j + threshold);
    289
    290            // the stripe may lead off of the table if s and t are of different sizes
    291            if (min > max) {
    292                return new LevenshteinResults(-1, 0, 0, 0);
    293            }
    294
    295            // ignore entry left of leftmost
    296            if (min > 1) {
    297                d[min - 1] = Integer.MAX_VALUE;
    298            }
    299
    300            // iterates through [min, max] in s
    301            for (int i = min; i <= max; i++) {
    302                if (left.charAt(i - 1) == rightJ) {
    303                    // diagonally left and up
    304                    d[i] = p[i - 1];
    305                } else {
    306                    // 1 + minimum of cell to the left, to the top, diagonally left and up
    307                    d[i] = 1 + Math.min(Math.min(d[i - 1], p[i]), p[i - 1]);
    308                }
    309                matrix[j][i] = d[i];
    310            }
    311
    312            // copy current distance counts to 'previous row' distance counts
    313            tempD = p;
    314            p = d;
    315            d = tempD;
    316        }
    317
    318        // if p[n] is greater than the threshold, there's no guarantee on it being the correct distance
    319        if (p[n] <= threshold) {
    320            return findDetailedResults(left, right, matrix, swapped);
    321        }
    322        return new LevenshteinResults(-1, 0, 0, 0);
    323    }
    324
    325    /**
    326     * Finds the Levenshtein distance between two Strings.
    327     *
    328     * <p>A higher score indicates a greater distance.</p>
    329     *
    330     * <p>The previous implementation of the Levenshtein distance algorithm
    331     * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p>
    332     *
    333     * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
    334     * which can occur when my Java implementation is used with very large strings.<br>
    335     * This implementation of the Levenshtein distance algorithm
    336     * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p>
    337     *
    338     * <pre>
    339     * unlimitedCompare(null, *)             = IllegalArgumentException
    340     * unlimitedCompare(*, null)             = IllegalArgumentException
    341     * unlimitedCompare("","")               = 0
    342     * unlimitedCompare("","a")              = 1
    343     * unlimitedCompare("aaapppp", "")       = 7
    344     * unlimitedCompare("frog", "fog")       = 1
    345     * unlimitedCompare("fly", "ant")        = 3
    346     * unlimitedCompare("elephant", "hippo") = 7
    347     * unlimitedCompare("hippo", "elephant") = 7
    348     * unlimitedCompare("hippo", "zzzzzzzz") = 8
    349     * unlimitedCompare("hello", "hallo")    = 1
    350     * </pre>
    351     *
    352     * @param left the first CharSequence, must not be null
    353     * @param right the second CharSequence, must not be null
    354     * @return result distance, or -1
    355     * @throws IllegalArgumentException if either CharSequence input is {@code null}
    356     */
    357    private static LevenshteinResults unlimitedCompare(CharSequence left, CharSequence right) {
    358        if (left == null || right == null) {
    359            throw new IllegalArgumentException("CharSequences must not be null");
    360        }
    361
    362        /*
    363           The difference between this impl. and the previous is that, rather
    364           than creating and retaining a matrix of size s.length() + 1 by t.length() + 1,
    365           we maintain two single-dimensional arrays of length s.length() + 1.  The first, d,
    366           is the 'current working' distance array that maintains the newest distance cost
    367           counts as we iterate through the characters of String s.  Each time we increment
    368           the index of String t we are comparing, d is copied to p, the second int[].  Doing so
    369           allows us to retain the previous cost counts as required by the algorithm (taking
    370           the minimum of the cost count to the left, up one, and diagonally up and to the left
    371           of the current cost count being calculated).  (Note that the arrays aren't really
    372           copied anymore, just switched...this is clearly much better than cloning an array
    373           or doing a System.arraycopy() each time  through the outer loop.)
    374
    375           Effectively, the difference between the two implementations is this one does not
    376           cause an out of memory condition when calculating the LD over two very large strings.
    377         */
    378
    379        int n = left.length(); // length of left
    380        int m = right.length(); // length of right
    381
    382        if (n == 0) {
    383            return new LevenshteinResults(m, m, 0, 0);
    384        }
    385        if (m == 0) {
    386            return new LevenshteinResults(n, 0, n, 0);
    387        }
    388        boolean swapped = false;
    389        if (n > m) {
    390            // swap the input strings to consume less memory
    391            final CharSequence tmp = left;
    392            left = right;
    393            right = tmp;
    394            n = m;
    395            m = right.length();
    396            swapped = true;
    397        }
    398
    399        int[] p = new int[n + 1]; // 'previous' cost array, horizontally
    400        int[] d = new int[n + 1]; // cost array, horizontally
    401        int[] tempD; //placeholder to assist in swapping p and d
    402        final int[][] matrix = new int[m + 1][n + 1];
    403
    404        // filling the first row and first column values in the matrix
    405        for (int index = 0; index <= n; index++) {
    406            matrix[0][index] = index;
    407        }
    408        for (int index = 0; index <= m; index++) {
    409            matrix[index][0] = index;
    410        }
    411
    412        // indexes into strings left and right
    413        int i; // iterates through left
    414        int j; // iterates through right
    415
    416        char rightJ; // jth character of right
    417
    418        int cost; // cost
    419        for (i = 0; i <= n; i++) {
    420            p[i] = i;
    421        }
    422
    423        for (j = 1; j <= m; j++) {
    424            rightJ = right.charAt(j - 1);
    425            d[0] = j;
    426
    427            for (i = 1; i <= n; i++) {
    428                cost = left.charAt(i - 1) == rightJ ? 0 : 1;
    429                // minimum of cell to the left+1, to the top+1, diagonally left and up +cost
    430                d[i] = Math.min(Math.min(d[i - 1] + 1, p[i] + 1), p[i - 1] + cost);
    431                //filling the matrix
    432                matrix[j][i] = d[i];
    433            }
    434
    435            // copy current distance counts to 'previous row' distance counts
    436            tempD = p;
    437            p = d;
    438            d = tempD;
    439        }
    440        return findDetailedResults(left, right, matrix, swapped);
    441    }
    442
    443    /**
    444     * Threshold.
    445     */
    446    private final Integer threshold;
    447
    448    /**
    449     * <p>
    450     * This returns the default instance that uses a version
    451     * of the algorithm that does not use a threshold parameter.
    452     * </p>
    453     *
    454     * @see LevenshteinDetailedDistance#getDefaultInstance()
    455     */
    456    public LevenshteinDetailedDistance() {
    457        this(null);
    458    }
    459
    460    /**
    461     * If the threshold is not null, distance calculations will be limited to a maximum length.
    462     *
    463     * <p>If the threshold is null, the unlimited version of the algorithm will be used.</p>
    464     *
    465     * @param threshold If this is null then distances calculations will not be limited. This may not be negative.
    466     */
    467    public LevenshteinDetailedDistance(final Integer threshold) {
    468        if (threshold != null && threshold < 0) {
    469            throw new IllegalArgumentException("Threshold must not be negative");
    470        }
    471        this.threshold = threshold;
    472    }
    473
    474    /**
    475     * Finds the Levenshtein distance between two Strings.
    476     *
    477     * <p>A higher score indicates a greater distance.</p>
    478     *
    479     * <p>The previous implementation of the Levenshtein distance algorithm
    480     * was from <a href="http://www.merriampark.com/ld.htm">http://www.merriampark.com/ld.htm</a></p>
    481     *
    482     * <p>Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
    483     * which can occur when my Java implementation is used with very large strings.<br>
    484     * This implementation of the Levenshtein distance algorithm
    485     * is from <a href="http://www.merriampark.com/ldjava.htm">http://www.merriampark.com/ldjava.htm</a></p>
    486     *
    487     * <pre>
    488     * distance.apply(null, *)             = IllegalArgumentException
    489     * distance.apply(*, null)             = IllegalArgumentException
    490     * distance.apply("","")               = 0
    491     * distance.apply("","a")              = 1
    492     * distance.apply("aaapppp", "")       = 7
    493     * distance.apply("frog", "fog")       = 1
    494     * distance.apply("fly", "ant")        = 3
    495     * distance.apply("elephant", "hippo") = 7
    496     * distance.apply("hippo", "elephant") = 7
    497     * distance.apply("hippo", "zzzzzzzz") = 8
    498     * distance.apply("hello", "hallo")    = 1
    499     * </pre>
    500     *
    501     * @param left the first string, must not be null
    502     * @param right the second string, must not be null
    503     * @return result distance, or -1
    504     * @throws IllegalArgumentException if either String input {@code null}
    505     */
    506    @Override
    507    public LevenshteinResults apply(final CharSequence left, final CharSequence right) {
    508        if (threshold != null) {
    509            return limitedCompare(left, right, threshold);
    510        }
    511        return unlimitedCompare(left, right);
    512    }
    513
    514    /**
    515     * Gets the distance threshold.
    516     *
    517     * @return The distance threshold
    518     */
    519    public Integer getThreshold() {
    520        return threshold;
    521    }
    522}