85de0ec1ca
Add TODOs. Fix some directory bugs when running scripts with a cwd not in the p4 workspace. TODO: make sure all scripts and options work when run outside p4 workspace. If user isn't logged in you can get weird errors later on in the pipeline, and without extra manually added prints, you wouldn't know you just need to log in. Added TODO: about detecting if we need to do a p4 login. Some of my stuff seems to have stopped working with later version of Python/p4, had to update string to byte string; no doubt more of these issues hiding. Haven't tested on python 2 in a while, do not consider these working there.
157 lines
No EOL
6.8 KiB
Python
157 lines
No EOL
6.8 KiB
Python
#!/usr/bin/python
|
|
# -*- coding: utf8 -*-
|
|
# author : Brian Ernst
|
|
# python_version : 2.7.6 and 3.4.0
|
|
# =================================
|
|
|
|
# todo: have a backup feature, make sure files are moved to the recycle bin or a temporary file.
|
|
# todo: add option of using send2trash
|
|
# todo: switch to faster method of calling p4 fstat on an entire directory and parsing it's output
|
|
# todo: allow logging output besides console output, or redirection altogether
|
|
|
|
from p4Helper import *
|
|
|
|
import time, traceback
|
|
|
|
|
|
#==============================================================
|
|
def main( args ):
|
|
start = time.time()
|
|
|
|
fail_if_no_p4()
|
|
|
|
#http://docs.python.org/library/optparse.html
|
|
parser = optparse.OptionParser( )
|
|
|
|
# TODO: Add dry-run option.
|
|
parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
|
|
parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 )
|
|
parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
|
|
parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=True )
|
|
parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
|
|
|
|
( options, args ) = parser.parse_args( args )
|
|
|
|
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
|
|
|
|
with Console( auto_flush_time=1 ) as c:
|
|
with P4Workspace( directory ):
|
|
# Files are added from .p4ignore
|
|
# Key is the file root, the value is the table of file regexes for that directory.
|
|
files_to_ignore = PDict()
|
|
|
|
processed_file_count = 0
|
|
processed_directory_count = 0
|
|
|
|
remove_file_count = 0
|
|
remove_dir_count = 0
|
|
warning_count = 0
|
|
error_count = 0
|
|
|
|
if not options.quiet:
|
|
c.writeflush( "\nCaching files in depot, this may take a little while..." )
|
|
|
|
# TODO: push this off to a thread and walk the directory so we get a headstart.
|
|
files_in_depot = get_client_set( directory )
|
|
|
|
if not options.quiet:
|
|
c.writeflush( "|Done." )
|
|
|
|
# TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?
|
|
|
|
# TODO: at this point join on both tasks to wait until they're done
|
|
|
|
# TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
|
|
# need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
|
|
# will at least need to redo how the ignore lists are handled for efficiencies sake.
|
|
|
|
if not options.quiet:
|
|
c.writeflush( "\nChecking " + directory)
|
|
for root, dirs, files in os.walk( directory ):
|
|
ignore_list = get_ignore_list( root, files_to_ignore )
|
|
|
|
if not options.quiet:
|
|
c.write( "|Checking " + os.path.relpath( root, directory ) )
|
|
|
|
for d in dirs:
|
|
processed_directory_count += 1
|
|
path = join( root, d )
|
|
rel_path = os.path.relpath( path, directory )
|
|
|
|
if match_in_ignore_list( path, ignore_list ):
|
|
# add option of using send2trash
|
|
if not options.quiet:
|
|
c.write( "| ignoring " + rel_path )
|
|
dirs.remove( d )
|
|
|
|
for f in files:
|
|
processed_file_count += 1
|
|
path = normpath( join( root, f ) )
|
|
|
|
if path not in files_in_depot:
|
|
if not options.quiet:
|
|
c.write( "| " + f + " is unversioned, removing it." )
|
|
try:
|
|
os.chmod( path, stat.S_IWRITE )
|
|
os.remove( path )
|
|
remove_file_count += 1
|
|
except OSError as ex:
|
|
c.writeflush( "| " + type( ex ).__name__ )
|
|
c.writeflush( "| " + repr( ex ) )
|
|
c.writeflush( "| ^ERROR^" )
|
|
|
|
error_count += 1
|
|
if not options.quiet:
|
|
c.write( "|Done." )
|
|
|
|
if not options.quiet:
|
|
c.write( os.linesep + "Removing empty directories...")
|
|
# remove empty directories in reverse order
|
|
for root, dirs, files in os.walk( directory, topdown=False ):
|
|
ignore_list = get_ignore_list( root, files_to_ignore )
|
|
|
|
for d in dirs:
|
|
processed_directory_count += 1
|
|
path = os.path.join( root, d )
|
|
rel_path = os.path.relpath( path, directory )
|
|
|
|
if match_in_ignore_list( path, ignore_list ):
|
|
# add option of using send2trash
|
|
if not options.quiet:
|
|
c.write( "| ignoring " + rel_path )
|
|
dirs.remove( d )
|
|
try:
|
|
os.rmdir(path)
|
|
remove_dir_count += 1
|
|
if not options.quiet:
|
|
c.write( "| " + rel_path + " was removed." )
|
|
except OSError:
|
|
# Fails on non-empty directory
|
|
pass
|
|
if not options.quiet:
|
|
c.write( "|Done." )
|
|
|
|
if not options.quiet:
|
|
output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " )
|
|
output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories")
|
|
|
|
output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " )
|
|
output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories")
|
|
|
|
if warning_count > 0:
|
|
output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" )
|
|
if error_count > 0:
|
|
output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )
|
|
|
|
end = time.time()
|
|
delta = end - start
|
|
output += "\nFinished in " + str(delta) + "s"
|
|
|
|
c.write( output )
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main( sys.argv )
|
|
except:
|
|
print( "\nUnexpected error!" )
|
|
traceback.print_exc( file = sys.stdout ) |