From 92d217371cbb0669fdf82ae4961a81989969ccef Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 13 May 2015 12:06:54 -0600 Subject: [PATCH] Fixed scripts up, improved logging so console has a waking thread now. Also fixed bug if console timer is too long it'll be killed off appropriately. --- p4Helper.py | 21 ++++++++++++++++++--- p4RemoveUnversioned.py | 2 +- p4SyncMissingFiles.py | 41 +++++++++++++++++++++++++++++++---------- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/p4Helper.py b/p4Helper.py index 5c20a3b..a82f845 100644 --- a/p4Helper.py +++ b/p4Helper.py @@ -81,11 +81,11 @@ def match_in_ignore_list( path, ignore_list ): #============================================================== def call_process( args ): - return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + return subprocess.call( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) def try_call_process( args, path=None ): try: - subprocess.check_output( args.split( ), shell=False, cwd=path, stderr=subprocess.STDOUT ) + subprocess.check_output( args, shell=False, cwd=path )#, stderr=subprocess.STDOUT ) return 0 except subprocess.CalledProcessError: return 1 @@ -113,7 +113,7 @@ def parse_info_from_command( args, value, path = None ): def get_p4_py_results( args, path = None ): results = [] - proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) + proc = subprocess.Popen( 'p4 -G ' + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) try: while True: output = marshal.load( proc.stdout ) @@ -286,6 +286,14 @@ class PDict( dict ): class Console( threading.Thread ): MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' ) + @staticmethod + def wake(thread): + thread.flush() + if not thread.shutting_down: + thread.wake_thread = threading.Timer(thread.auto_flush_time / 1000.0, Console.wake, [thread]) + thread.wake_thread.daemon = True + thread.wake_thread.start() + # auto_flush_time is time in milliseconds since last flush to trigger a flush when writing def __init__( self, auto_flush_num = None, auto_flush_time = None ): threading.Thread.__init__( self ) @@ -296,6 +304,9 @@ class Console( threading.Thread ): self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1 self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1 self.shutting_down = False + self.wake_thread = None + if self.auto_flush_time > 0: + Console.wake(self) def write( self, data, pid = None ): pid = pid if pid is not None else threading.current_thread().ident @@ -319,6 +330,10 @@ class Console( threading.Thread ): return self def __exit__( self, type, value, tb ): + self.shutting_down = True + if self.wake_thread: + self.wake_thread.cancel() + self.wake_thread.join() self.queue.put( ( Console.MSG.SHUTDOWN, ) ) self.queue.join( ) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index a816cc4..eaaa960 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -35,7 +35,7 @@ def main( args ): directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) - with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: + with Console( auto_flush_time=1 ) as c: with P4Workspace( directory ): # Files are added from .p4ignore # Key is the file root, the value is the table of file regexes for that directory. diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py index 28da004..c5b6d33 100644 --- a/p4SyncMissingFiles.py +++ b/p4SyncMissingFiles.py @@ -32,10 +32,10 @@ class P4SyncMissing: directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) - with Console( auto_flush_time=1000 ) as c: + with Console( auto_flush_time=1 ) as c: with P4Workspace( directory ): if not options.quiet: - c.writeflush( "Preparing to sync missing files..." ) + c.writeflush( "Retreiving missing files..." ) c.writeflush( " Setting up threads..." ) # Setup threading @@ -44,18 +44,29 @@ class P4SyncMissing: def shutdown( data ): return False def sync( files ): + files_len = len(files) files_flat = ' '.join('"' + f + '"' for f in files) + + if options.verbose: + files_len = len(files) + if files_len > 1: + c.write( " Syncing batch of " + str(len(files)) + " ...") + for f in files: + c.write( " " + os.path.relpath( f, directory ) ) + else: + for f in files: + c.write( " Syncing " + os.path.relpath( f, directory ) + " ..." ) + ret = -1 count = 0 while ret != 0 and count < 2: ret = try_call_process( "p4 sync -f " + files_flat ) count += 1 - #if ret != 0 and not options.quiet: - # c.write("Failed, trying again to sync " + files_flat) + if ret != 0 and not options.quiet: + c.write("Failed, trying again to sync " + files_flat) if ret != 0: - pass - #if not options.quiet: - # c.write("Failed to sync " + files_flat) + if not options.quiet: + c.write("Failed to sync " + files_flat) else: if not options.quiet: files_len = len(files) @@ -74,13 +85,17 @@ class P4SyncMissing: thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads count = 0 + total = 0 self.queue = multiprocessing.JoinableQueue( ) for i in range( thread_count ): t = Worker( c, self.queue, commands ) + t.daemon = True threads.append( t ) t.start( ) + c.writeflush( " Done." ) + make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False command = "p4 fstat ..." @@ -105,6 +120,7 @@ class P4SyncMissing: file_type = None file_type_last = None + # todo: use fewer threads, increase bucket size and use p4 threading class Bucket: def __init__(self, limit): self.queue = [] @@ -118,7 +134,7 @@ class P4SyncMissing: self.buckets = {} self.buckets[file_type_text] = Bucket(10) - self.buckets[file_type_binary] = Bucket(1) + self.buckets[file_type_binary] = Bucket(2) def push_queued(bucket): if bucket.queue_size == 0: @@ -153,11 +169,12 @@ class P4SyncMissing: if any(file_action == a for a in rejected_actions): file_action = None else: + total += 1 if os.path.exists( client_file ): file_action = None elif line.startswith( clientFile_tag ): - client_file = normpath( line[ len( clientFile_tag ) : ].strip( ) ) + client_file = line[ len( clientFile_tag ) : ].strip( ) if make_drive_upper: drive, path = splitdrive( client_file ) client_file = ''.join( [ drive.upper( ), path ] ) @@ -176,7 +193,8 @@ class P4SyncMissing: c.write(line)#log as error if not options.quiet: - c.writeflush( " Checking " + str(count) + " file(s), now waiting for threads..." ) + c.writeflush( " Done. Checked " + str(total) + " file(s)." ) + c.writeflush( " Queued " + str(count) + " file(s), now waiting for threads..." ) for i in range( thread_count ): self.queue.put( ( WRK.SHUTDOWN, None ) ) @@ -184,6 +202,9 @@ class P4SyncMissing: for t in threads: t.join( ) + if not options.quiet: + print( " Done." ) + if not options.quiet: print( "Done." )