p4Scripts/p4RemoveUnversioned.py

#!/usr/bin/python
# -*- coding: utf8 -*-
# author              : Brian Ernst
# python_version      : 2.7.6 and 3.4.0
# =================================

# todo: have a backup feature, make sure files are moved to the recycle bin or a temporary file.
# todo: add option of using send2trash
# todo: switch to faster method of calling p4 fstat on an entire directory and parsing it's output
# todo: allow logging output besides console output, or redirection altogether

from p4Helper import *

import time, traceback


#==============================================================
def main( args ):
    start = time.time()

    fail_if_no_p4()

    #http://docs.python.org/library/optparse.html
    parser = optparse.OptionParser( )

    parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
    parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 )
    parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
    parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=True )
    parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )

    ( options, args ) = parser.parse_args( args )

    directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )

    with Console( auto_flush_time=1 ) as c:
        with P4Workspace( directory ):
            # Files are added from .p4ignore
            # Key is the file root, the value is the table of file regexes for that directory.
            files_to_ignore = PDict()

            processed_file_count = 0
            processed_directory_count = 0
            
            remove_file_count = 0
            remove_dir_count = 0
            warning_count = 0
            error_count = 0

            if not options.quiet:
                c.writeflush( "\nCaching files in depot, this may take a little while..." )

            # TODO: push this off to a thread and walk the directory so we get a headstart.
            files_in_depot = get_client_set( directory )

            if not options.quiet:
                c.writeflush( "|Done." )

            # TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?

            # TODO: at this point join on both tasks to wait until they're done

            # TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
            #       need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
            #       will at least need to redo how the ignore lists are handled for efficiencies sake.

            if not options.quiet:
                c.writeflush( "\nChecking " + directory)
            for root, dirs, files in os.walk( directory ):
                ignore_list = get_ignore_list( root, files_to_ignore )

                if not options.quiet:
                    c.write( "|Checking " + os.path.relpath( root, directory ) )

                for d in dirs:
                    processed_directory_count += 1
                    path = join( root, d )
                    rel_path = os.path.relpath( path, directory )

                    if match_in_ignore_list( path, ignore_list ):
                        # add option of using send2trash
                        if not options.quiet:
                            c.write( "| ignoring " + rel_path )
                        dirs.remove( d )

                for f in files:
                    processed_file_count += 1
                    path = normpath( join( root, f ) )

                    if path not in files_in_depot:
                        if not options.quiet:
                            c.write( "| " + f + " is unversioned, removing it." )
                        try:
                            os.chmod( path, stat.S_IWRITE )
                            os.remove( path )
                            remove_file_count += 1
                        except OSError as ex:
                            c.writeflush( "|  " + type( ex ).__name__ )
                            c.writeflush( "|  " + repr( ex ) )
                            c.writeflush( "|  ^ERROR^" )

                            error_count += 1
            if not options.quiet:
                c.write( "|Done." )

            if not options.quiet:
                c.write( os.linesep + "Removing empty directories...")
            # remove empty directories in reverse order
            for root, dirs, files in os.walk( directory, topdown=False ):
                ignore_list = get_ignore_list( root, files_to_ignore )

                for d in dirs:
                    processed_directory_count += 1
                    path = os.path.join( root, d )
                    rel_path = os.path.relpath( path, directory )

                    if match_in_ignore_list( path, ignore_list ):
                        # add option of using send2trash
                        if not options.quiet:
                            c.write( "| ignoring " + rel_path )
                        dirs.remove( d )
                    try:
                        os.rmdir(path)
                        remove_dir_count += 1
                        if not options.quiet:
                            c.write( "| " + rel_path + " was removed." )
                    except OSError:
                        # Fails on non-empty directory
                        pass
            if not options.quiet:
                c.write( "|Done." )

            if not options.quiet:
                output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " )
                output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories")
                
                output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " )
                output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories")

                if warning_count > 0:
                    output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" )
                if error_count > 0:
                    output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )

                end = time.time()
                delta = end - start
                output += "\nFinished in " + str(delta) + "s"

                c.write( output )

if __name__ == "__main__":
    try:
        main( sys.argv )
    except:
        print( "\nUnexpected error!" )
        traceback.print_exc( file = sys.stdout )