#!/usr/bin/python # -*- coding: utf8 -*- # author : Brian Ernst # python_version : 2.7.6 and 3.4.0 # ================================= # TODO: setup batches before pushing to threads and use p4 --parallel # http://www.perforce.com/perforce/r14.2/manuals/cmdref/p4_sync.html from p4Helper import * import time, traceback #============================================================== def main( args ): start = time.clock() fail_if_no_p4() #http://docs.python.org/library/optparse.html parser = optparse.OptionParser( ) parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None ) parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 ) parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False ) parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False ) parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False ) ( options, args ) = parser.parse_args( args ) directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: with P4Workspace( directory ): if not options.quiet: c.writeflush( "Preparing to sync missing files..." ) c.write( " Setting up threads..." ) # Setup threading WRK = enum( 'SHUTDOWN', 'SYNC' ) def shutdown( data ): return False def sync( data ): if data is not None and not os.path.exists( data ): try_call_process( "p4 sync -f " + data ) if not options.quiet: c.write( " Synced " + data ) return True commands = { WRK.SHUTDOWN : shutdown, WRK.SYNC : sync } threads = [ ] thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads queue = multiprocessing.JoinableQueue( ) for i in range( thread_count ): t = Worker( c, queue, commands ) threads.append( t ) t.start( ) make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False command = "p4 fstat ..." if not options.quiet: c.writeflush( " Checking files in depot, this may take some time for large depots..." ) proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory ) clientFile_tag = "... clientFile " headAction_tag = "... headAction " # http://www.perforce.com/perforce/r12.1/manuals/cmdref/fstat.html accepted_actions = [ 'add', 'edit', 'branch', 'move/add', 'move\\add', 'integrate', 'import', 'archive' ] #currently not checked rejected_actions = [ 'delete', 'move/delete', 'move\\delete', 'purge' ] client_file = None for line in proc.stdout: line = get_str_from_process_stdout( line ) if client_file and line.startswith( headAction_tag ): action = normpath( line[ len( headAction_tag ) : ].strip( ) ) if not any(action == a for a in rejected_actions): if options.verbose: c.write( " Checking " + os.path.relpath( local_path, directory ) ) queue.put( ( WRK.SYNC, local_path ) ) if line.startswith( clientFile_tag ): client_file = None local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) ) if make_drive_upper: drive, path = splitdrive( local_path ) client_file = ''.join( [ drive.upper( ), path ] ) if len(line.rstrip()) == 0: client_file = None proc.wait( ) for line in proc.stderr: if "no such file" in line: continue #raise Exception(line) c.write(line)#log as error if not options.quiet: c.writeflush( " Pushed work, now waiting for threads..." ) for i in range( thread_count ): queue.put( ( WRK.SHUTDOWN, None ) ) for t in threads: t.join( ) if not options.quiet: c.write( "Done." ) end = time.clock() delta = end - start output = "\nFinished in " + str(delta) + "s" c.writeflush( output ) if __name__ == "__main__": try: main( sys.argv ) except: print( "\nUnexpected error!" ) traceback.print_exc( file = sys.stdout )