137 lines
5.1 KiB
Python
137 lines
5.1 KiB
Python
#!/usr/bin/python
|
|
# -*- coding: utf8 -*-
|
|
# author : Brian Ernst
|
|
# python_version : 2.7.6 and 3.4.0
|
|
# =================================
|
|
|
|
# TODO: setup batches before pushing to threads and use p4 --parallel
|
|
# http://www.perforce.com/perforce/r14.2/manuals/cmdref/p4_sync.html
|
|
|
|
from p4Helper import *
|
|
|
|
import time, traceback
|
|
|
|
|
|
#==============================================================
|
|
def main( args ):
|
|
start = time.clock()
|
|
|
|
fail_if_no_p4()
|
|
|
|
#http://docs.python.org/library/optparse.html
|
|
parser = optparse.OptionParser( )
|
|
|
|
parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
|
|
parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 )
|
|
parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
|
|
parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
|
|
parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
|
|
|
|
( options, args ) = parser.parse_args( args )
|
|
|
|
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
|
|
|
|
with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
|
|
with P4Workspace( directory ):
|
|
if not options.quiet:
|
|
c.writeflush( "Preparing to sync missing files..." )
|
|
c.write( " Setting up threads..." )
|
|
|
|
# Setup threading
|
|
WRK = enum( 'SHUTDOWN', 'SYNC' )
|
|
|
|
def shutdown( data ):
|
|
return False
|
|
def sync( data ):
|
|
if data is not None and not os.path.exists( data ):
|
|
subprocess.check_output( "p4 sync -f \"" + data + "\"", shell=False, cwd=None )
|
|
if not options.quiet:
|
|
c.write( " Synced " + os.path.relpath( data, directory ) )
|
|
return True
|
|
|
|
commands = {
|
|
WRK.SHUTDOWN : shutdown,
|
|
WRK.SYNC : sync
|
|
}
|
|
|
|
threads = [ ]
|
|
thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
|
|
|
|
queue = multiprocessing.JoinableQueue( )
|
|
|
|
for i in range( thread_count ):
|
|
t = Worker( c, queue, commands )
|
|
threads.append( t )
|
|
t.start( )
|
|
|
|
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
|
|
|
|
command = "p4 fstat ..."
|
|
|
|
if not options.quiet:
|
|
c.writeflush( " Checking files in depot, this may take some time for large depots..." )
|
|
|
|
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory )
|
|
|
|
clientFile_tag = "... clientFile "
|
|
headAction_tag = "... headAction "
|
|
|
|
# http://www.perforce.com/perforce/r12.1/manuals/cmdref/fstat.html
|
|
accepted_actions = [ 'add', 'edit', 'branch', 'move/add', 'move\\add', 'integrate', 'import', 'archive' ] #currently not checked
|
|
rejected_actions = [ 'delete', 'move/delete', 'move\\delete', 'purge' ]
|
|
|
|
client_file = None
|
|
|
|
for line in proc.stdout:
|
|
line = get_str_from_process_stdout( line )
|
|
|
|
if client_file and line.startswith( headAction_tag ):
|
|
action = normpath( line[ len( headAction_tag ) : ].strip( ) )
|
|
if not any(action == a for a in rejected_actions):
|
|
if options.verbose:
|
|
c.write( " Checking " + os.path.relpath( local_path, directory ) )
|
|
# TODO: directories should be batched and synced in parallel
|
|
queue.put( ( WRK.SYNC, local_path ) )
|
|
|
|
if line.startswith( clientFile_tag ):
|
|
client_file = None
|
|
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
|
|
if make_drive_upper:
|
|
drive, path = splitdrive( local_path )
|
|
client_file = ''.join( [ drive.upper( ), path ] )
|
|
|
|
if len(line.rstrip()) == 0:
|
|
client_file = None
|
|
|
|
proc.wait( )
|
|
|
|
for line in proc.stderr:
|
|
if "no such file" in line:
|
|
continue
|
|
#raise Exception(line)
|
|
c.write(line)#log as error
|
|
|
|
if not options.quiet:
|
|
c.writeflush( " Pushed work, now waiting for threads..." )
|
|
|
|
for i in range( thread_count ):
|
|
queue.put( ( WRK.SHUTDOWN, None ) )
|
|
|
|
for t in threads:
|
|
t.join( )
|
|
|
|
if not options.quiet:
|
|
c.write( "Done." )
|
|
|
|
end = time.clock()
|
|
delta = end - start
|
|
output = "\nFinished in " + str(delta) + "s"
|
|
|
|
c.writeflush( output )
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main( sys.argv )
|
|
except:
|
|
print( "\nUnexpected error!" )
|
|
traceback.print_exc( file = sys.stdout ) |