spacesavers2_catalog

#!/usr/bin/env python3

from src.VersionCheck import version_check
from src.VersionCheck import __version__
from src.utils import *

version_check()

# import required modules
import textwrap
import tqdm
import sys
from src.FileDetails import FileDetails
from multiprocessing import Pool
import argparse
from pathlib import Path


def task(f):
    fd = FileDetails()
    # print(f"Initiating {f}")
    fd.initialize(
        f,
        buffersize=args.buffersize,
        thresholdsize=args.ignoreheadersize,
        tb=args.buffersize,
        sed=sed,
        bottomhash=args.bottomhash,
        st_block_byte_size=args.st_block_byte_size,
    )
    # print(f"Returning {f}")
    return fd

def process(fd,broken_links,outfh,geezerage,geezersize,geezers):
    uid = fd.get_userid()
    if fd.get_type() == "L": # broken link
        if not uid in broken_links: broken_links[uid] = list()
        broken_links[uid].append(fd.get_filepath())
    else:
        result = "%s" % (fd)
        if not result == "":
            outfh.write(f"{result}\n")
            if fd.get_type() == "f":
                age = fd.get_age()
                size = fd.get_size()
                if age > geezerage and size > geezersize:
                    x = list()
                    x.append("{0:.2f} yrs".format(age/365))
                    x.append(fd.get_size_human_readable())
                    x.append(fd.get_filepath())
                    if not uid in geezers: geezers[uid] = list()
                    geezers[uid].append("\t".join(x))

def main():
    elog = textwrap.dedent(
        """\
    Version:
        {}
    Example:
        > spacesavers2_catalog -f /path/to/folder -p 56 -e -l -g
        """.format(
            __version__
        )
    )
    parser = argparse.ArgumentParser(
        description="spacesavers2_catalog: get per file info.",
        epilog=elog,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "-f",
        "--folder",
        dest="folder",
        required=True,
        type=str,
        help="spacesavers2_catalog will be run on all files in this folder and its subfolders",
    )
    parser.add_argument(
        "-p",
        "--threads",
        dest="threads",
        required=False,
        type=int,
        default=4,
        help="number of threads to be used (default 4)",
    )
    parser.add_argument(
        "-b",
        "--buffersize",
        dest="buffersize",
        required=False,
        type=int,
        default=128 * 1024,
        help="buffersize for xhash creation (default=128 * 1028 bytes)",
    )
    parser.add_argument(
        "-i",
        "--ignoreheadersize",
        dest="ignoreheadersize",
        required=False,
        type=int,
        default=1024 * 1024 * 1024,
        help="this sized header of the file is ignored before extracting buffer of buffersize for xhash creation (only for special extensions files) default = 1024 * 1024 * 1024 bytes",
    )
    parser.add_argument(
        "-x",
        "--se",
        dest="se",
        required=False,
        type=str,
        default="bam,bai,bigwig,bw,csi",
        help="comma separated list of special extensions (default=bam,bai,bigwig,bw,csi)",
    )
    parser.add_argument(
        "-s",
        "--st_block_byte_size",
        dest="st_block_byte_size",
        required=False,
        default=512,
        type=int,
        help="st_block_byte_size on current filesystem (default 512)",
    )
    parser.add_argument(
        "-o",
        "--outfile",
        dest="outfile",
        required=False,
        type=str,
        help="outfile ... catalog file .. by default output is printed to screen",
    )
    parser.add_argument(
        "-e",
        "--bottomhash",
        dest="bottomhash",
        required=False,
        action=argparse.BooleanOptionalAction,
        help="separately calculated second hash for the bottom/end of the file.",
    )
    parser.add_argument(
        "-l",
        "--brokenlink",
        dest="brokenlink",
        required=False,
        action=argparse.BooleanOptionalAction,
        help="output per-user broken links list.",
    )
    parser.add_argument(
        "-g",
        "--geezers",
        dest="geezers",
        required=False,
        action=argparse.BooleanOptionalAction,
        help="output per-user geezer files list.",
    )
    parser.add_argument(
        "-q",
        "--quiet",
        dest="quiet",
        required=False,
        action=argparse.BooleanOptionalAction,
        help="Do not show progress",
    )
    parser.add_argument(
        "-a",
        "--geezerage",
        dest="geezerage",
        required=False,
        default= 5 * 365,
        type=int,
        help="age in days to be considered a geezer file (default 5yrs ... 5 * 365).",
    )
    parser.add_argument(
        "-z",
        "--geezersize",
        dest="geezersize",
        required=False,
        default= 10 * 1024 * 1024,
        type=int,
        help="minimum size in bytes of geezer file to be reported (default 10MiB ... 10 * 1024 * 1024).",
    )
    parser.add_argument("-v", "--version", action="version", version=__version__)

    global args
    args = parser.parse_args()

    tqdm_disable = False
    if args.quiet: tqdm_disable = True
    
    global sed
    sed = dict()
    for s in args.se.split(","):
        sed[s] = 1

    folder = args.folder
    p = Path(folder).absolute()
    dirs = [p]

    broken_links = dict()
    geezers = dict()

    if args.outfile:
        outfh = open(args.outfile, 'w')
    else:
        outfh = sys.stdout

    with Pool(processes=args.threads) as pool:
        for fd in tqdm.tqdm(pool.imap_unordered(task, scantree(p,dirs)),disable=tqdm_disable):
            process(fd,broken_links,outfh,args.geezerage,args.geezersize,geezers)

    with Pool(processes=args.threads) as pool:
        for fd in tqdm.tqdm(pool.imap_unordered(task, dirs),disable=tqdm_disable):
            process(fd,broken_links,outfh,args.geezerage,args.geezersize,geezers)

    if args.outfile:
        outfh.close()
        if args.brokenlink: # spit out broken links
            for uid in broken_links:
                username = get_username_groupname(uid)
                outfilename = args.outfile
                if outfilename.endswith(".txt"):
                    outfilename = outfilename[:-4]
                outfh_broken_links = open(outfilename + "." + username + ".brokenlinks.txt", 'w')
                for l in broken_links[uid]:
                    outfh_broken_links.write(f"{l}\n")
                outfh_broken_links.close()
        if args.geezers:
            for uid in geezers:
                username = get_username_groupname(uid)
                outfilename = args.outfile
                if outfilename.endswith(".txt"):
                    outfilename = outfilename[:-4]
                outfh_geezers = open(outfilename + "." + username + ".geezers.txt", 'w')
                for l in geezers[uid]:
                    outfh_geezers.write(f"{l}\n")
                outfh_geezers.close()
if __name__ == "__main__":
    main()