#!/usr/bin/python3
# Copyright (C) 2023 J.F.Dockes
#
# License: GPL 2.1
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

"""Report the longest documents in a Recoll index (by term count), and the biggest 
directories (by sum of term counts for contained documents).
This should give a pretty good estimate of what consumes space in your index.
You will need to install the Xapian and Recoll Python3 bindings (e.g. python3-xapian, 
python3-recoll on Debian or Ubuntu)"""

import sys
import os
import getopt

import xapian
import recoll.rclconfig as rclconfig

# Default count of biggest files we show
nfilesprinted = 10
# Default depth of paths for which we compute occupation. e.g. /home/me/doc/science would be
# at depth 4
dirdepth = 4
# Default count of directories for which we print the totals
ndirsprinted = 20

def msg(s):
    print(f"{s}", file=sys.stderr)
    
# Retrieve named value(s) from a recoll document data record.
# The record format is a sequence of nm=value lines
def get_attributes(xdb, xdocid, flds, decode=True):
    #msg(f"get_attributes xdocid {xdocid} flds {flds} decode {decode}")
    doc = xdb.get_document(xdocid)
    data = doc.get_data()
    res = []
    for fld in flds:
        s = data.find(fld + b"=")
        if s == -1:
            res.append(None)
        else:
            e = data.find(b"\n", s)
            if decode:
                res.append(data[s+len(fld)+1:e].decode('UTF-8'))
            else:
                res.append(data[s+len(fld)+1:e])
    return res

# Sample code. Not called and does nothing at the moment. Sample code for looking at doc terms in
# more detail
def doc_details(xdb, xdocid):
    #xdoc = xdb.get_document(xdocid)
    #for term in xdoc.termlist():
        #msg(f"TERM {term.term}")
        #for position in xdb.positionlist(xdocid, term.term):
            #poscount +=1
    #msg(f"Positions count {poscount}")
    pass
    

if __name__ == '__main__':
    # Note: the -c option is only for command line tests, the recoll process always sets the config
    # in RECOLL_CONFDIR
    def usage(f=sys.stderr):
        prog = os.path.basename(sys.argv[0])
        print(f"Usage: {prog} [OPTION]...", file=f)
        print(f" -h, --help         show this help", file=f)
        print(f" -c, --config       select Recoll configuration directory", file=f)
        print(f" -n, --files-count  set the size of the file list"
              f" (default {nfilesprinted})", file=f)
        print(f" -d, --depth        set the depth of the selected directories (default {dirdepth})",
              file=f)
        print(f" -N, --dirs-count   set the size of the directory list (default {ndirsprinted})",
              file=f)
        sys.exit(1)

    confdir = None
    try:
        options, args = getopt.getopt(sys.argv[1:], "hc:d:N:n:",
                                      ["help", "config=", "files-count=", "depth=", "dirs-count="])
    except Exception as err:
        print(err, file=sys.stderr)
        usage()
    for o,a in options:
        if o in ("-h", "--help"):
            usage(sys.stdout)
        elif o in ("-c", "--config"):
            confdir = a
        elif o in ("-d", "--depth"):
            dirdepth = int(a)
        elif o in ("-N", "--dirs-count"):
            ndirsprinted = int(a)
        elif o in ("-n", "--files-count"):
            nfilesprinted = int(a)
        
    if len(args) != 0:
        usage()

    rclconfig = rclconfig.RclConfig(argcnf=confdir)
    confdir = rclconfig.getConfDir()
    dbdir = rclconfig.getDbDir()
    xdb = xapian.Database(dbdir)

    alldoclengths = []

    # Walk the Xapian index whole document list (postlist for empty term). Take note of the document
    # term counts.
    lastxdocid = xdb.get_lastdocid()
    for doc in xdb.postlist(""):
        xdocid = int(doc.docid)
        if xdocid % 10000 == 0:
            msg(f"Walking whole Xapian term list: docid {xdocid}/{lastxdocid}")
        try:
            url = get_attributes(xdb, xdocid, [b"url",], decode=False)[0]
        except Exception as ex:
            msg(f"get_attributes failed: {ex}")
            continue
        alldoclengths.append((doc.doclength, url))

    # Sort by document term count, highest first
    alldoclengths.sort(reverse=True)

    # Walk the sorted list, print the "nfilesprinted" longest documents sizes and paths, accumulate
    # the sizes for directories at depth "dirdepth"
    dirsizes = {}
    i = 0
    for e in alldoclengths:
        path = e[1][7:]
        if i < nfilesprinted:
            print(f"{e[0]} {path}")
            i += 1
        path = os.path.dirname(path)
        #print(f"PATH {path}")
        l = path.split(b"/")
        if len(l) < dirdepth:
            dir = path
        else:
            dir = b"/".join(l[0:dirdepth+1])
        #print(f"DIR {dir}")
        if dir in dirsizes:
            dirsizes[dir] += e[0]
        else:
            dirsizes[dir] = e[0]

    # Print the biggest directories
    print("\nDirectory sizes:")
    dirsizes  = sorted(dirsizes.items(), key = lambda entry: entry[1], reverse = True)
    for dir, sz in dirsizes[0:ndirsprinted]:
        print(f"{sz} {dir}")
