Fixed scripts up, improved logging so console has a waking thread now.

Also fixed bug if console timer is too long it'll be killed off
appropriately.
This commit is contained in:
unknown 2015-05-13 12:06:54 -06:00
parent ea14f96d76
commit 92d217371c
3 changed files with 50 additions and 14 deletions

View File

@ -81,11 +81,11 @@ def match_in_ignore_list( path, ignore_list ):
#==============================================================
def call_process( args ):
return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
return subprocess.call( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
def try_call_process( args, path=None ):
try:
subprocess.check_output( args.split( ), shell=False, cwd=path, stderr=subprocess.STDOUT )
subprocess.check_output( args, shell=False, cwd=path )#, stderr=subprocess.STDOUT )
return 0
except subprocess.CalledProcessError:
return 1
@ -113,7 +113,7 @@ def parse_info_from_command( args, value, path = None ):
def get_p4_py_results( args, path = None ):
results = []
proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
proc = subprocess.Popen( 'p4 -G ' + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
try:
while True:
output = marshal.load( proc.stdout )
@ -286,6 +286,14 @@ class PDict( dict ):
class Console( threading.Thread ):
MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' )
@staticmethod
def wake(thread):
thread.flush()
if not thread.shutting_down:
thread.wake_thread = threading.Timer(thread.auto_flush_time / 1000.0, Console.wake, [thread])
thread.wake_thread.daemon = True
thread.wake_thread.start()
# auto_flush_time is time in milliseconds since last flush to trigger a flush when writing
def __init__( self, auto_flush_num = None, auto_flush_time = None ):
threading.Thread.__init__( self )
@ -296,6 +304,9 @@ class Console( threading.Thread ):
self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1
self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
self.shutting_down = False
self.wake_thread = None
if self.auto_flush_time > 0:
Console.wake(self)
def write( self, data, pid = None ):
pid = pid if pid is not None else threading.current_thread().ident
@ -319,6 +330,10 @@ class Console( threading.Thread ):
return self
def __exit__( self, type, value, tb ):
self.shutting_down = True
if self.wake_thread:
self.wake_thread.cancel()
self.wake_thread.join()
self.queue.put( ( Console.MSG.SHUTDOWN, ) )
self.queue.join( )

View File

@ -35,7 +35,7 @@ def main( args ):
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
with Console( auto_flush_time=1 ) as c:
with P4Workspace( directory ):
# Files are added from .p4ignore
# Key is the file root, the value is the table of file regexes for that directory.

View File

@ -32,10 +32,10 @@ class P4SyncMissing:
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
with Console( auto_flush_time=1000 ) as c:
with Console( auto_flush_time=1 ) as c:
with P4Workspace( directory ):
if not options.quiet:
c.writeflush( "Preparing to sync missing files..." )
c.writeflush( "Retreiving missing files..." )
c.writeflush( " Setting up threads..." )
# Setup threading
@ -44,18 +44,29 @@ class P4SyncMissing:
def shutdown( data ):
return False
def sync( files ):
files_len = len(files)
files_flat = ' '.join('"' + f + '"' for f in files)
if options.verbose:
files_len = len(files)
if files_len > 1:
c.write( " Syncing batch of " + str(len(files)) + " ...")
for f in files:
c.write( " " + os.path.relpath( f, directory ) )
else:
for f in files:
c.write( " Syncing " + os.path.relpath( f, directory ) + " ..." )
ret = -1
count = 0
while ret != 0 and count < 2:
ret = try_call_process( "p4 sync -f " + files_flat )
count += 1
#if ret != 0 and not options.quiet:
# c.write("Failed, trying again to sync " + files_flat)
if ret != 0 and not options.quiet:
c.write("Failed, trying again to sync " + files_flat)
if ret != 0:
pass
#if not options.quiet:
# c.write("Failed to sync " + files_flat)
if not options.quiet:
c.write("Failed to sync " + files_flat)
else:
if not options.quiet:
files_len = len(files)
@ -74,13 +85,17 @@ class P4SyncMissing:
thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
count = 0
total = 0
self.queue = multiprocessing.JoinableQueue( )
for i in range( thread_count ):
t = Worker( c, self.queue, commands )
t.daemon = True
threads.append( t )
t.start( )
c.writeflush( " Done." )
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
command = "p4 fstat ..."
@ -105,6 +120,7 @@ class P4SyncMissing:
file_type = None
file_type_last = None
# todo: use fewer threads, increase bucket size and use p4 threading
class Bucket:
def __init__(self, limit):
self.queue = []
@ -118,7 +134,7 @@ class P4SyncMissing:
self.buckets = {}
self.buckets[file_type_text] = Bucket(10)
self.buckets[file_type_binary] = Bucket(1)
self.buckets[file_type_binary] = Bucket(2)
def push_queued(bucket):
if bucket.queue_size == 0:
@ -153,11 +169,12 @@ class P4SyncMissing:
if any(file_action == a for a in rejected_actions):
file_action = None
else:
total += 1
if os.path.exists( client_file ):
file_action = None
elif line.startswith( clientFile_tag ):
client_file = normpath( line[ len( clientFile_tag ) : ].strip( ) )
client_file = line[ len( clientFile_tag ) : ].strip( )
if make_drive_upper:
drive, path = splitdrive( client_file )
client_file = ''.join( [ drive.upper( ), path ] )
@ -176,7 +193,8 @@ class P4SyncMissing:
c.write(line)#log as error
if not options.quiet:
c.writeflush( " Checking " + str(count) + " file(s), now waiting for threads..." )
c.writeflush( " Done. Checked " + str(total) + " file(s)." )
c.writeflush( " Queued " + str(count) + " file(s), now waiting for threads..." )
for i in range( thread_count ):
self.queue.put( ( WRK.SHUTDOWN, None ) )
@ -184,6 +202,9 @@ class P4SyncMissing:
for t in threads:
t.join( )
if not options.quiet:
print( " Done." )
if not options.quiet:
print( "Done." )