From 8d425d6413be5f6a41cccd0ad5cb0dae0d54e615 Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 9 May 2014 15:21:56 -0600 Subject: [PATCH 01/32] Catch except in file iteration so you can continue processing remaining files. The next changes will be ground shaking, a lot should be changing, performance should increase significantly. --- p4RemoveUnversioned.py | 28 ++++++++++++++++++++++------ python2.7.exe.stackdump | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 python2.7.exe.stackdump diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 1d30a2c..8c12050 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -155,7 +155,17 @@ class Worker( threading.Thread ): self.console.write( "Working on " + directory ) - dir_contents = os.listdir( directory ) + try: + dir_contents = os.listdir( directory ) + except OSError as ex: + self.console.write( "| " + type( ex ).__name__ ) + # args can be anything, can't guarantee they'll convert to a string + #self.console.write( "| " + ' '.join( [ str( arg ) for arg in ex.args ] ) ) + self.console.write( "| " + repr( ex ) ) + self.console.write( "|ERROR." ) + self.console.flush( ) + self.queue.task_done( ) + continue if p4_ignore in dir_contents: file_regexes = [] @@ -184,9 +194,10 @@ class Worker( threading.Thread ): proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory ) (out, err) = proc.communicate() except Exception as ex: - self.console.write( "| " + type( ex ) ) - self.console.write( "| " + ex.args ) - self.console.write( "| " + ex ) + self.console.write( "| " + type( ex ).__name__ ) + # args can be anything, can't guarantee they'll convert to a string + #self.console.write( "| " + ' '.join( [ str( arg ) for arg in ex.args ] ) ) + self.console.write( "| " + repr( ex ) ) self.console.write( "|ERROR." ) self.console.flush( ) self.queue.task_done( ) @@ -226,8 +237,13 @@ class Worker( threading.Thread ): continue self.console.write( "| " + file + " is unversioned, removing it." ) - os.chmod( path, stat.S_IWRITE ) - os.remove( path ) + try: + os.chmod( path, stat.S_IWRITE ) + os.remove( path ) + except OSError as ex: + self.console.write( "| " + type( ex ).__name__ ) + self.console.write( "| " + repr( ex ) ) + self.console.write( "|ERROR." ) self.console.write( "|Done." ) self.console.flush( ) diff --git a/python2.7.exe.stackdump b/python2.7.exe.stackdump new file mode 100644 index 0000000..2050f78 --- /dev/null +++ b/python2.7.exe.stackdump @@ -0,0 +1,16 @@ +Stack trace: +Frame Function Args +000FD4FC6C0 0018006F733 (FFFFFFFFFFFFFFFF, 006002C2510, 005CCDD1850, 005CCDD1870) +00000000006 00180070C6A (000FD4FC8A0, 00000000000, 0000000035C, 00000000000) +000FD4FC8A0 00180118778 (00000000000, 00000000000, 000FD4FCA20, 00600000003) +00000000041 0018011587E (00180133B2D, 000000002CC, 00000000000, 00000000000) +00000000000 00180115D4B (001801362C0, 006002C2270, 000775D19A1, 00000000006) +00000000000 00180115F1C (00000000000, 0008FB10544, 7FEFD6464DA, 006002C2318) +00000000000 001801161DF (0018013B2FB, 00600000001, 001801CA2C0, 000FD4FD640) +00000000000 00180144816 (00180135C7F, 006002C2270, 006002C2270, 006002C2270) +00000000000 001800BFCE3 (0018013460D, 00000000000, 005CCDB7127, 006002C2318) +00000000000 001801629FC (00000000000, 00000000000, 00000000000, 006002C2270) +00000000000 00180136AEF (00000000000, 00000000000, 00000000000, 00000000000) +00000000000 00180136334 (000003A0000, 00000000000, 00000000000, 00000000000) +00000000000 001800C373B (000003A0000, 00000000000, 00000000000, 00000000000) +End of stack trace From d7ff7a6646459d2323b0a16163c6f9aa0f31804d Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 9 May 2014 15:24:12 -0600 Subject: [PATCH 02/32] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index abd326d..ce65e1e 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,7 @@ Removes unversioned files from perforce repository. Script is in beta, though it This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex. **Files are currently permanently deleted, so use this at your own risk.** + +Raw Tests +=================== +On a 133GB Directory with 15,700 Folders and 153,415 Files, the script ran for 11m and 16.35s. This will only get better. From bcff97c22ced27c377801eed25b595277f4578bc Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 9 May 2014 15:40:13 -0600 Subject: [PATCH 03/32] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ce65e1e..7c7c053 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,6 @@ This script does parse __.p4ignore__ ignore files, compiles the fields as regex, Raw Tests =================== -On a 133GB Directory with 15,700 Folders and 153,415 Files, the script ran for 11m and 16.35s. This will only get better. +With Python 2.7.6 on a 133GB Directory with 15,700 Folders and 153,415 Files, the script ran for 11m and 16.35s. This will only get better. + +With Python 3.4.0 on the same directory, the script ran for 3m and 00.86s. At this point the directory was already stripped of unversioned files, but that's not the longest part of the process, so this version of Python is just that much faster. I'll have more concrete tests later when things are changing less in code. From beab6428c11ddac6b5b84a3efa979ce0a6ac5816 Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 9 May 2014 17:19:09 -0600 Subject: [PATCH 04/32] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 7c7c053..02b16e7 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,12 @@ This script does parse __.p4ignore__ ignore files, compiles the fields as regex, **Files are currently permanently deleted, so use this at your own risk.** +**This is currently Windows only, but it will be updated to work on Linux** + Raw Tests =================== With Python 2.7.6 on a 133GB Directory with 15,700 Folders and 153,415 Files, the script ran for 11m and 16.35s. This will only get better. With Python 3.4.0 on the same directory, the script ran for 3m and 00.86s. At this point the directory was already stripped of unversioned files, but that's not the longest part of the process, so this version of Python is just that much faster. I'll have more concrete tests later when things are changing less in code. + +Notes From c175b21dcf6ee0450ecb8673cb24b02088153a24 Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 9 May 2014 17:19:44 -0600 Subject: [PATCH 05/32] Grabs depot tree first hand to make looping through directory faster. The big catch right now is, this method is single threaded, I haven't made it multi-threaded yet, but it definitely looks like it can benefit from it. --- p4RemoveUnversioned.py | 152 +++++++++++++++++++++++++---------------- 1 file changed, 92 insertions(+), 60 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 8c12050..9aa5ef1 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -11,7 +11,7 @@ # todo: buffer output, after exceeding a certain amount print to the output. # todo: allow logging output besides console output, or redirection altogether -import inspect, multiprocessing, optparse, os, re, stat, subprocess, sys, threading, traceback +import inspect, multiprocessing, optparse, os, platform, re, stat, subprocess, sys, threading, traceback # trying ntpath, need to test on linux import ntpath @@ -36,10 +36,27 @@ p4_ignore = ".p4ignore" main_pid = os.getpid( ) +#if os.name == 'nt' or sys.platform == 'cygwin' def basename( path ): + # TODO: import based on platform + # https://docs.python.org/2/library/os.path.html + # posixpath for UNIX-style paths + # ntpath for Windows paths + # macpath for old-style MacOS paths + # os2emxpath for OS/2 EMX paths + #return os.path.basename( path ) return ntpath.basename( path ) +def normpath( path ): + return ntpath.normpath( path ) + +def join( patha, pathb ): + return ntpath.join( patha, pathb ) + +def splitdrive( path ): + return ntpath.splitdrive( path ) + def get_ignore_list( path, files_to_ignore ): # have to split path and test top directory dirs = path.split( os.sep ) @@ -60,6 +77,31 @@ def match_in_ignore_list( path, ignore_list ): return True return False +# Keep these in mind if you have issues: +# https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime +# https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running +def get_client_set( path ): + files = set( [ ] ) + + make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False + + command = "p4 fstat ..." + + proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=None, cwd=path ) + for line in proc.stdout: + clientFile_tag = "... clientFile " + if not line.startswith( clientFile_tag ): + continue + + local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) ) + if make_drive_upper: + drive, path = splitdrive( local_path ) + local_path = ''.join( [ drive.upper( ), path ] ) + + files.add( local_path ) + + return files + class PTable( list ): def __init__( self, *args ): list.__init__( self, args ) @@ -73,20 +115,27 @@ class PDict( dict ): class Console( threading.Thread ): MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' ) - def __init__( self ): + def __init__( self, auto_flush_num = None, auto_flush_time = None ): threading.Thread.__init__( self ) self.buffers = {} self.running = True self.queue = multiprocessing.JoinableQueue( ) + self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1 + self.auto_flush_time = auto_flush_time if auto_flush_time is not None else -1 - def write( self, data ): - self.queue.put( ( Console.MSG.WRITE, os.getpid(), data ) ) + def write( self, data, pid = None ): + self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) ) - def flush( self ): - self.queue.put( ( Console.MSG.FLUSH, os.getpid() ) ) + def writeflush( self, data, pid = None ): + pid = pid if pid is not None else os.getpid() + self.queue.put( ( Console.MSG.WRITE, pid, data ) ) + self.queue.put( ( Console.MSG.FLUSH, pid ) ) - def clear( self ): - self.queue.put( ( Console.MSG.CLEAR, os.getpid() ) ) + def flush( self, pid = None ): + self.queue.put( ( Console.MSG.FLUSH, pid if pid is not None else os.getpid() ) ) + + def clear( self, pid = None ): + self.queue.put( ( Console.MSG.CLEAR, pid if pid is not None else os.getpid() ) ) def __enter__( self ): self.start( ) @@ -103,7 +152,7 @@ class Console( threading.Thread ): if event == Console.MSG.SHUTDOWN: # flush remaining buffers before shutting down - for ( pid, buffer ) in self.buffers.iteritems( ): + for ( pid, buffer ) in self.buffers.items( ): for line in buffer: print( line ) self.buffers.clear( ) @@ -116,6 +165,9 @@ class Console( threading.Thread ): if pid not in self.buffers: self.buffers[ pid ] = [] self.buffers[ pid ].append( s ) + + if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) > self.auto_flush_num: + self.flush( pid ) elif event == Console.MSG.FLUSH: pid = data[ 1 ] if pid in self.buffers: @@ -170,12 +222,13 @@ class Worker( threading.Thread ): if p4_ignore in dir_contents: file_regexes = [] # Should automatically ignore .p4ignore even if it's not specified, otherwise it'll be deleted. - path = os.path.join( directory, p4_ignore ) + path = join( directory, p4_ignore ) with open( path ) as f: for line in f: new_line = remove_comment( line.strip( ) ) if len( new_line ) > 0: - file_regexes.append( re.compile( os.path.join( re.escape( directory + os.sep ), new_line ) ) ) + # doesn't look quite right, fix it: + file_regexes.append( re.compile( join( re.escape( directory + os.sep ), new_line ) ) ) self.console.write( "| Appending ignores from " + path ) with self.files_to_ignore.mutex: @@ -216,13 +269,13 @@ class Worker( threading.Thread ): if base == "*" or len(base) == 0: # Directory is empty, we could delete it now continue - path = os.path.join( directory, base ) + path = join( directory, base ) if not os.path.isdir( path ): files.append( base ) for content in dir_contents: - path = os.path.join( directory, content ) + path = join( directory, content ) if os.path.isdir( path ): if match_in_ignore_list( path, ignore_list ): self.console.write( "| Ignoring " + content ) @@ -230,7 +283,7 @@ class Worker( threading.Thread ): self.queue.put( ( MSG.PARSE_DIRECTORY, path ) ) for file in files: - path = os.path.join( directory, file ) + path = join( directory, file ) if match_in_ignore_list( path, ignore_list ): self.console.write( "| Ignoring " + path ) @@ -261,66 +314,45 @@ def main( args ): parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None ) parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 ) - parser.add_option( "-q", "--quiet", action="store_false", dest="quiet", default=False ) + parser.add_option( "-q", "--quiet", action="store_false", dest="quiet", help="This overrides verbose", default=False ) parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=True ) ( options, args ) = parser.parse_args( ) - root_full_path = os.getcwd( ) + directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) - # Files are added from .p4ignore - # Key is the file root, the value is the table of file regexes for that directory. - files_to_ignore = PDict() + with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: + c.writeflush( "Caching files in depot..." ) + files_in_depot = get_client_set( directory ) - # make sure script doesn't delete itself - with files_to_ignore.mutex: - files_to_ignore[ root_full_path ] = [ re.compile( re.escape( os.path.join( root_full_path, basename( __file__ ) ) ) ) ] + c.writeflush( "Checking " + directory) + for root, dirs, files in os.walk( directory ): + ignore_list = PDict()#get_ignore_list( root, files_to_ignore ) - # Setup threading - threads = [] - thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads - - queue = multiprocessing.JoinableQueue( ) - - with Console() as c: - for i in range( thread_count ): - t = Worker( c, queue, files_to_ignore ) - threads.append( t ) - t.start( ) - - if len( threads ) == 1: - print( "Spawned %s thread." % len( threads ) ) - else: - print( "Spawned %s threads." % len( threads ) ) - - queue.put( ( MSG.PARSE_DIRECTORY, options.directory if options.directory is not None else os.getcwd( ) ) ) - queue.join( ) - - for i in range( thread_count ): - queue.put( ( MSG.SHUTDOWN, None ) ) - - print( os.linesep + "Removing empty directories...") - # remove empty directories in reverse order - for root, dirs, files in os.walk( root_full_path, topdown=False ): - ignore_list = get_ignore_list( root, files_to_ignore ) + c.write( "|Checking " + root ) for d in dirs: - path = os.path.join( root, d ) + path = join( root, d ) if match_in_ignore_list( path, ignore_list ): # add option of using send2trash - print( "| ignoring " + d ) + c.write( "| ignoring " + d ) dirs.remove( d ) - try: - os.rmdir(path) - print( "| " + d + " was removed." ) - except OSError: - # Fails on non-empty directory - pass - print( "|Done." ) - for t in threads: - t.join( ) + for f in files: + path = normpath( join( root, f ) ) + + if path not in files_in_depot: + c.write( "| " + path ) + c.write( "| " + f + " is unversioned, removing it." ) + #try: + # os.chmod( path, stat.S_IWRITE ) + # os.remove( path ) + #except OSError as ex: + # c.writeflush( "| " + type( ex ).__name__ ) + # c.writeflush( "| " + repr( ex ) ) + # c.writeflush( "|ERROR." ) + c.write( "|Done." ) if __name__ == "__main__": try: From 55a5e41b008eefa84c12822dc9cc3f6805723f9f Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 9 May 2014 17:36:49 -0600 Subject: [PATCH 06/32] Improved the auto flushing, made it time and buffer size based. In case a specific directory was taking a while, I changed it to auto flush after a specified period of time. Right now autoflush is automatically disabled, you have to enable it when creating the console. TODO: I'll probably hook the console up to the stdout and stderr so you can use ordinary print statements, we'll see. This is desirable for easily hooking it into an existing module. --- p4RemoveUnversioned.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 9aa5ef1..668dacb 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -11,7 +11,7 @@ # todo: buffer output, after exceeding a certain amount print to the output. # todo: allow logging output besides console output, or redirection altogether -import inspect, multiprocessing, optparse, os, platform, re, stat, subprocess, sys, threading, traceback +import datetime, inspect, multiprocessing, optparse, os, re, stat, subprocess, sys, threading, traceback # trying ntpath, need to test on linux import ntpath @@ -115,13 +115,15 @@ class PDict( dict ): class Console( threading.Thread ): MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' ) + # auto_flush_time is time in milliseconds since last flush to trigger a flush when writing def __init__( self, auto_flush_num = None, auto_flush_time = None ): threading.Thread.__init__( self ) self.buffers = {} + self.buffer_write_times = {} self.running = True self.queue = multiprocessing.JoinableQueue( ) self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1 - self.auto_flush_time = auto_flush_time if auto_flush_time is not None else -1 + self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1 def write( self, data, pid = None ): self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) ) @@ -164,9 +166,13 @@ class Console( threading.Thread ): if pid not in self.buffers: self.buffers[ pid ] = [] + if pid not in self.buffer_write_times: + self.buffer_write_times[ pid ] = datetime.datetime.now( ) self.buffers[ pid ].append( s ) - if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) > self.auto_flush_num: + if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num: + self.flush( pid ) + elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time: self.flush( pid ) elif event == Console.MSG.FLUSH: pid = data[ 1 ] @@ -174,6 +180,7 @@ class Console( threading.Thread ): for line in self.buffers[ pid ]: print( line ) self.buffers.pop( pid, None ) + self.buffer_write_times[ pid ] = datetime.datetime.now( ) elif event == Console.MSG.CLEAR: pid = data[ 1 ] if pid in self.buffers: From 0dcd14a73bf91fb663691f20ee2c856cc34dfe1a Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 9 May 2014 19:11:23 -0600 Subject: [PATCH 07/32] Working in Python 2.7.4 and Python 3.4.0, HOWEVER, Console isn't exiting correctly. --- p4RemoveUnversioned.py | 141 ++++++----------------------------------- 1 file changed, 18 insertions(+), 123 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 668dacb..1c61548 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -77,6 +77,13 @@ def match_in_ignore_list( path, ignore_list ): return True return False +use_bytearray_str_conversion = type( b"str" ) is not str +def get_str_from_process_stdout( line ): + if use_bytearray_str_conversion: + return ''.join( map( chr, line ) ) + else: + return line + # Keep these in mind if you have issues: # https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running @@ -89,6 +96,8 @@ def get_client_set( path ): proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=None, cwd=path ) for line in proc.stdout: + line = get_str_from_process_stdout( line ) + clientFile_tag = "... clientFile " if not line.startswith( clientFile_tag ): continue @@ -112,6 +121,7 @@ class PDict( dict ): dict.__init__( self, args ) self.mutex = multiprocessing.Semaphore( ) +# TODO: Create a child thread for triggering autoflush events class Console( threading.Thread ): MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' ) @@ -124,6 +134,7 @@ class Console( threading.Thread ): self.queue = multiprocessing.JoinableQueue( ) self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1 self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1 + self.shutting_down = False def write( self, data, pid = None ): self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) ) @@ -158,7 +169,11 @@ class Console( threading.Thread ): for line in buffer: print( line ) self.buffers.clear( ) + self.buffer_write_times.clear( ) self.queue.task_done( ) + + print(self.queue.qsize()) + print(self.queue.empty()) break elif event == Console.MSG.WRITE: @@ -174,6 +189,7 @@ class Console( threading.Thread ): self.flush( pid ) elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time: self.flush( pid ) + # TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time elif event == Console.MSG.FLUSH: pid = data[ 1 ] if pid in self.buffers: @@ -188,128 +204,6 @@ class Console( threading.Thread ): self.queue.task_done( ) -class Worker( threading.Thread ): - def __init__( self, console, queue, files_to_ignore ): - threading.Thread.__init__( self ) - - self.console = console - self.queue = queue - self.files_to_ignore = files_to_ignore - - def run( self ): - while True: - ( cmd, data ) = self.queue.get( ) - - if cmd == MSG.SHUTDOWN: - self.queue.task_done( ) - self.console.flush( ) - break - - if cmd != MSG.PARSE_DIRECTORY or data is None: - self.console.flush( ) - self.queue.task_done( ) - continue - - directory = data - - self.console.write( "Working on " + directory ) - - try: - dir_contents = os.listdir( directory ) - except OSError as ex: - self.console.write( "| " + type( ex ).__name__ ) - # args can be anything, can't guarantee they'll convert to a string - #self.console.write( "| " + ' '.join( [ str( arg ) for arg in ex.args ] ) ) - self.console.write( "| " + repr( ex ) ) - self.console.write( "|ERROR." ) - self.console.flush( ) - self.queue.task_done( ) - continue - - if p4_ignore in dir_contents: - file_regexes = [] - # Should automatically ignore .p4ignore even if it's not specified, otherwise it'll be deleted. - path = join( directory, p4_ignore ) - with open( path ) as f: - for line in f: - new_line = remove_comment( line.strip( ) ) - if len( new_line ) > 0: - # doesn't look quite right, fix it: - file_regexes.append( re.compile( join( re.escape( directory + os.sep ), new_line ) ) ) - - self.console.write( "| Appending ignores from " + path ) - with self.files_to_ignore.mutex: - if directory not in self.files_to_ignore: - self.files_to_ignore[ directory ] = [] - self.files_to_ignore[ directory ].extend( file_regexes ) - - - ignore_list = get_ignore_list( directory, self.files_to_ignore ) - - - files = [] - command = "p4 fstat *" - - try: - proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory ) - (out, err) = proc.communicate() - except Exception as ex: - self.console.write( "| " + type( ex ).__name__ ) - # args can be anything, can't guarantee they'll convert to a string - #self.console.write( "| " + ' '.join( [ str( arg ) for arg in ex.args ] ) ) - self.console.write( "| " + repr( ex ) ) - self.console.write( "|ERROR." ) - self.console.flush( ) - self.queue.task_done( ) - continue - - for line in err.decode('utf-8').split( os.linesep ): - if len( line ) == 0: - continue - - # # dirty hack that grabs the filename from the ends of the printed out (not err) "depo_path - local_path" - # # I could use regex to verify the expected string, but that will just slow us down. - # base = basename( line ) - i = line.rfind( ' - ') - if i >= 0: - base = line[ : i ] - if base == "*" or len(base) == 0: - # Directory is empty, we could delete it now - continue - path = join( directory, base ) - - if not os.path.isdir( path ): - files.append( base ) - - for content in dir_contents: - path = join( directory, content ) - if os.path.isdir( path ): - if match_in_ignore_list( path, ignore_list ): - self.console.write( "| Ignoring " + content ) - else: - self.queue.put( ( MSG.PARSE_DIRECTORY, path ) ) - - for file in files: - path = join( directory, file ) - - if match_in_ignore_list( path, ignore_list ): - self.console.write( "| Ignoring " + path ) - continue - - self.console.write( "| " + file + " is unversioned, removing it." ) - try: - os.chmod( path, stat.S_IWRITE ) - os.remove( path ) - except OSError as ex: - self.console.write( "| " + type( ex ).__name__ ) - self.console.write( "| " + repr( ex ) ) - self.console.write( "|ERROR." ) - - self.console.write( "|Done." ) - self.console.flush( ) - - self.queue.task_done( ) - def main( args ): # check requirements if os.system( 'p4 > Nul' ) != 0: @@ -334,7 +228,7 @@ def main( args ): c.writeflush( "Checking " + directory) for root, dirs, files in os.walk( directory ): - ignore_list = PDict()#get_ignore_list( root, files_to_ignore ) + ignore_list = {}#get_ignore_list( root, files_to_ignore ) c.write( "|Checking " + root ) @@ -359,6 +253,7 @@ def main( args ): # c.writeflush( "| " + type( ex ).__name__ ) # c.writeflush( "| " + repr( ex ) ) # c.writeflush( "|ERROR." ) + c.write( "|Done." ) if __name__ == "__main__": From 4435a36bedc5dd2aed97106e7b05ba6420c55060 Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 13 May 2014 14:08:16 -0600 Subject: [PATCH 08/32] Made script obey quiet option. Added file and error count to print at end. Also made sure the error output gets piped and doesn't show up in console. However, we shouldn't ignore any error output, this should be accounted for and properly logged. So, this is a TODO. --- p4RemoveUnversioned.py | 52 ++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 1c61548..f16adf8 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -94,7 +94,7 @@ def get_client_set( path ): command = "p4 fstat ..." - proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=None, cwd=path ) + proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) for line in proc.stdout: line = get_str_from_process_stdout( line ) @@ -222,39 +222,61 @@ def main( args ): directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) + remove_count = 0 + warning_count = 0 + error_count = 0 + with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: - c.writeflush( "Caching files in depot..." ) + if not options.quiet: + c.writeflush( "Caching files in depot..." ) files_in_depot = get_client_set( directory ) - c.writeflush( "Checking " + directory) + if not options.quiet: + c.writeflush( "Checking " + directory) for root, dirs, files in os.walk( directory ): ignore_list = {}#get_ignore_list( root, files_to_ignore ) - c.write( "|Checking " + root ) + if not options.quiet: + c.write( "|Checking " + root ) for d in dirs: path = join( root, d ) if match_in_ignore_list( path, ignore_list ): # add option of using send2trash - c.write( "| ignoring " + d ) + if not options.quiet: + c.write( "| ignoring " + d ) dirs.remove( d ) for f in files: path = normpath( join( root, f ) ) if path not in files_in_depot: - c.write( "| " + path ) - c.write( "| " + f + " is unversioned, removing it." ) - #try: - # os.chmod( path, stat.S_IWRITE ) - # os.remove( path ) - #except OSError as ex: - # c.writeflush( "| " + type( ex ).__name__ ) - # c.writeflush( "| " + repr( ex ) ) - # c.writeflush( "|ERROR." ) + if not options.quiet: + c.write( "| " + f + " is unversioned, removing it." ) + try: + os.chmod( path, stat.S_IWRITE ) + os.remove( path ) + remove_count += 1 + except OSError as ex: + c.writeflush( "| " + type( ex ).__name__ ) + c.writeflush( "| " + repr( ex ) ) + c.writeflush( "|ERROR." ) - c.write( "|Done." ) + error_count += 1 + + if not options.quiet: + c.write( "|Done." ) + + output = "\nRemoved " + str( remove_count ) + " file/s" + + if warning_count > 0: + output += " w/ " + str( warning_count ) + " warning/s" + + if error_count > 0: + output += " w/ " + str( error_count ) + " errors/s" + + c.write( output + "." ) if __name__ == "__main__": try: From 3d110652994bb573947fb935a0b91cd2438523bf Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 13 May 2014 16:56:13 -0600 Subject: [PATCH 09/32] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 02b16e7..49e0903 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,10 @@ With Python 2.7.6 on a 133GB Directory with 15,700 Folders and 153,415 Files, th With Python 3.4.0 on the same directory, the script ran for 3m and 00.86s. At this point the directory was already stripped of unversioned files, but that's not the longest part of the process, so this version of Python is just that much faster. I'll have more concrete tests later when things are changing less in code. +**UPDATE** +To give you an idea of the improvement, running the new script in the testing branch with Python 3.4.0 has a runtime of 3m and 44.44s, singlethreaded. The previous test was done with 100 threads. So, once I make this multi-threaded, it will be blazing. + + Notes +=================== +Besides making the new script multi-threaded, I'm looking at a way of improving the os.walk speed, I see there's a betterwalk module I can take advantage of. I noticed when iterating directories for deletion it's super fast, but iterating files makes it super slow for some reason; so, I want to look into this. From 865eaa243da384ca35c6bab62d3c381f3d210d4d Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 13 May 2014 19:08:53 -0600 Subject: [PATCH 10/32] Removed creation of NUL file, annoying to get rid of. Also changed error formatting a little. --- p4RemoveUnversioned.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index f16adf8..67ec4b8 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -206,7 +206,7 @@ class Console( threading.Thread ): def main( args ): # check requirements - if os.system( 'p4 > Nul' ) != 0: + if os.system( 'p4 -V' ) != 0: print( 'Perforce Command-line Client(p4) is required for this script.' ) sys.exit( 1 ) @@ -259,9 +259,9 @@ def main( args ): os.remove( path ) remove_count += 1 except OSError as ex: - c.writeflush( "| " + type( ex ).__name__ ) - c.writeflush( "| " + repr( ex ) ) - c.writeflush( "|ERROR." ) + c.writeflush( "| " + type( ex ).__name__ ) + c.writeflush( "| " + repr( ex ) ) + c.writeflush( "| ^ERROR^" ) error_count += 1 @@ -274,7 +274,7 @@ def main( args ): output += " w/ " + str( warning_count ) + " warning/s" if error_count > 0: - output += " w/ " + str( error_count ) + " errors/s" + output += " w/ " + str( error_count ) + " error/s" c.write( output + "." ) From fd419089be7945c60cc90a359451af37ef7c4858 Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 13 May 2014 20:18:16 -0600 Subject: [PATCH 11/32] See description. Why does this have to be so short? Removed excess input of polling p4. Fixed quiet output. Added directory removal back in. Made the output a little nicer, added singular and plural strings, also added directory total output. --- p4RemoveUnversioned.py | 58 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 67ec4b8..913a650 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -77,6 +77,9 @@ def match_in_ignore_list( path, ignore_list ): return True return False +def call_process( args ): + return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + use_bytearray_str_conversion = type( b"str" ) is not str def get_str_from_process_stdout( line ): if use_bytearray_str_conversion: @@ -84,6 +87,9 @@ def get_str_from_process_stdout( line ): else: return line +def singular_pulural( val, singular, plural ): + return singular if val == 1 else plural + # Keep these in mind if you have issues: # https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running @@ -109,6 +115,8 @@ def get_client_set( path ): files.add( local_path ) + # TODO: check error to see if the path is not in the client view. Prompt anyway? + return files class PTable( list ): @@ -206,7 +214,7 @@ class Console( threading.Thread ): def main( args ): # check requirements - if os.system( 'p4 -V' ) != 0: + if call_process( 'p4 -V' ) != 0: print( 'Perforce Command-line Client(p4) is required for this script.' ) sys.exit( 1 ) @@ -215,14 +223,19 @@ def main( args ): parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None ) parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 ) - parser.add_option( "-q", "--quiet", action="store_false", dest="quiet", help="This overrides verbose", default=False ) + parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False ) parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=True ) ( options, args ) = parser.parse_args( ) directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) - remove_count = 0 + # Files are added from .p4ignore + # Key is the file root, the value is the table of file regexes for that directory. + files_to_ignore = PDict() + + remove_file_count = 0 + remove_dir_count = 0 warning_count = 0 error_count = 0 @@ -234,7 +247,7 @@ def main( args ): if not options.quiet: c.writeflush( "Checking " + directory) for root, dirs, files in os.walk( directory ): - ignore_list = {}#get_ignore_list( root, files_to_ignore ) + ignore_list = get_ignore_list( root, files_to_ignore ) if not options.quiet: c.write( "|Checking " + root ) @@ -257,24 +270,49 @@ def main( args ): try: os.chmod( path, stat.S_IWRITE ) os.remove( path ) - remove_count += 1 + remove_file_count += 1 except OSError as ex: c.writeflush( "| " + type( ex ).__name__ ) c.writeflush( "| " + repr( ex ) ) c.writeflush( "| ^ERROR^" ) error_count += 1 - if not options.quiet: c.write( "|Done." ) - output = "\nRemoved " + str( remove_count ) + " file/s" + if not options.quiet: + c.write( os.linesep + "Removing empty directories...") + # remove empty directories in reverse order + for root, dirs, files in os.walk( directory, topdown=False ): + ignore_list = get_ignore_list( root, files_to_ignore ) + + for d in dirs: + path = os.path.join( root, d ) + + if match_in_ignore_list( path, ignore_list ): + # add option of using send2trash + if not options.quiet: + c.write( "| ignoring " + d ) + dirs.remove( d ) + try: + os.rmdir(path) + remove_dir_count += 1 + if not options.quiet: + c.write( "| " + d + " was removed." ) + except OSError: + # Fails on non-empty directory + pass + if not options.quiet: + c.write( "|Done." ) + + if not options.quiet: + output = "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " ) + output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories") if warning_count > 0: - output += " w/ " + str( warning_count ) + " warning/s" - + output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" ) if error_count > 0: - output += " w/ " + str( error_count ) + " error/s" + output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" ) c.write( output + "." ) From 59e010d682c28dc8a31d824f71894966ac0e38dc Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 13 May 2014 20:33:11 -0600 Subject: [PATCH 12/32] Added a warning note for large depots. --- p4RemoveUnversioned.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 913a650..30c44c1 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -241,7 +241,7 @@ def main( args ): with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: if not options.quiet: - c.writeflush( "Caching files in depot..." ) + c.writeflush( "Caching files in depot, this may take a little while..." ) files_in_depot = get_client_set( directory ) if not options.quiet: From 3ffdd761474ad404d0830ac4436f3d63f26f0dc0 Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 13 May 2014 20:45:55 -0600 Subject: [PATCH 13/32] Added basic worker thread back in, and TODO comments for multi-threading this new script. --- p4RemoveUnversioned.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 30c44c1..8fd2a8d 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -212,6 +212,34 @@ class Console( threading.Thread ): self.queue.task_done( ) +class Worker( threading.Thread ): + def __init__( self, console, queue, files_to_ignore ): + threading.Thread.__init__( self ) + + self.console = console + self.queue = queue + self.files_to_ignore = files_to_ignore + + def run( self ): + while True: + ( cmd, data ) = self.queue.get( ) + + if cmd == MSG.SHUTDOWN: + self.queue.task_done( ) + self.console.flush( ) + break + + if cmd != MSG.PARSE_DIRECTORY or data is None: + self.console.flush( ) + self.queue.task_done( ) + continue + + directory = data + + # add threading stuffs + + self.queue.task_done( ) + def main( args ): # check requirements if call_process( 'p4 -V' ) != 0: @@ -242,8 +270,18 @@ def main( args ): with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: if not options.quiet: c.writeflush( "Caching files in depot, this may take a little while..." ) + + # TODO: push this off to a thread and walk the directory so we get a headstart. files_in_depot = get_client_set( directory ) + # TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory? + + # TODO: at this point join on both tasks to wait until they're done + + # TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each. + # need to figure out the best way to do this since the ignore list needs to be properly built for each directory; + # will at least need to redo how the ignore lists are handled for efficiencies sake. + if not options.quiet: c.writeflush( "Checking " + directory) for root, dirs, files in os.walk( directory ): From 6236ead3385422f490f31fb7f54bdab7071e32b9 Mon Sep 17 00:00:00 2001 From: Brian Date: Wed, 14 May 2014 19:09:46 -0600 Subject: [PATCH 14/32] Adding new worker run type --- p4RemoveUnversioned.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 8fd2a8d..bb8805c 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -29,7 +29,7 @@ def enum(*sequential, **named): enums = dict(zip(sequential, range(len(sequential))), **named) return type('Enum', (), enums) -MSG = enum('SHUTDOWN', 'PARSE_DIRECTORY') +MSG = enum('SHUTDOWN', 'PARSE_DIRECTORY', 'RUN_FUNCTION') p4_ignore = ".p4ignore" @@ -212,6 +212,19 @@ class Console( threading.Thread ): self.queue.task_done( ) +class Task( threading.Event ): + def __init__( data, cmd = None ): + threading.Event.__init__( self ) + + self.cmd = cmd if cmd is None MSG.RUN_FUNCTION + self.data = data + + def isDone( self ): + return self.isSet() + + def join( self ): + self.wait( ) + class Worker( threading.Thread ): def __init__( self, console, queue, files_to_ignore ): threading.Thread.__init__( self ) @@ -225,8 +238,11 @@ class Worker( threading.Thread ): ( cmd, data ) = self.queue.get( ) if cmd == MSG.SHUTDOWN: - self.queue.task_done( ) self.console.flush( ) + self.queue.task_done( ) + break + + if cmd == MSG.RUN_FUNCTION: break if cmd != MSG.PARSE_DIRECTORY or data is None: From 06b0cbe426ae2fc7b3dfc1ecef8c9dd347da4125 Mon Sep 17 00:00:00 2001 From: "U-ILLFONIC\\bernst" Date: Wed, 13 Aug 2014 17:09:19 -0600 Subject: [PATCH 15/32] Adding huge improvements. There are still a few more to make to account for computers not setup correctly, but it's functional. Still has the occasional console hang bug. Now also prints out run time. There is one new minor bug, reverting back to the previously set client view. --- p4RemoveUnversioned.py | 164 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 18 deletions(-) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index bb8805c..28bae51 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -11,7 +11,7 @@ # todo: buffer output, after exceeding a certain amount print to the output. # todo: allow logging output besides console output, or redirection altogether -import datetime, inspect, multiprocessing, optparse, os, re, stat, subprocess, sys, threading, traceback +import datetime, inspect, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading, time, traceback # trying ntpath, need to test on linux import ntpath @@ -80,6 +80,13 @@ def match_in_ignore_list( path, ignore_list ): def call_process( args ): return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) +def try_call_process( args, path=None ): + try: + subprocess.check_output( args.split( ), shell=False, cwd=path ) + return 0 + except subprocess.CalledProcessError: + return 1 + use_bytearray_str_conversion = type( b"str" ) is not str def get_str_from_process_stdout( line ): if use_bytearray_str_conversion: @@ -90,6 +97,33 @@ def get_str_from_process_stdout( line ): def singular_pulural( val, singular, plural ): return singular if val == 1 else plural +def parse_info_from_command( args, value, path = None ): + """ + + :rtype : string + """ + proc = subprocess.Popen( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) + for line in proc.stdout: + line = get_str_from_process_stdout( line ) + + if not line.startswith( value ): + continue + return line[ len( value ) : ].strip( ) + return None + +def get_p4_py_results( args, path = None ): + results = [] + proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) + try: + while True: + output = marshal.load( proc.stdout ) + results.append( output ) + except EOFError: + pass + finally: + proc.stdout.close() + return results + # Keep these in mind if you have issues: # https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running @@ -115,10 +149,33 @@ def get_client_set( path ): files.add( local_path ) - # TODO: check error to see if the path is not in the client view. Prompt anyway? + proc.wait( ) + + for line in proc.stderr: + raise Exception(line) return files +def get_client_root( ): + """ + + :rtype : string + """ + command = "p4 info" + + proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + for line in proc.stdout: + line = get_str_from_process_stdout( line ) + + clientFile_tag = "Client root: " + if not line.startswith( clientFile_tag ): + continue + + local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) ) + + return local_path + return None + class PTable( list ): def __init__( self, *args ): list.__init__( self, args ) @@ -212,18 +269,18 @@ class Console( threading.Thread ): self.queue.task_done( ) -class Task( threading.Event ): - def __init__( data, cmd = None ): - threading.Event.__init__( self ) +# class Task( threading.Event ): + # def __init__( data, cmd = None ): + # threading.Event.__init__( self ) - self.cmd = cmd if cmd is None MSG.RUN_FUNCTION - self.data = data + # self.cmd = cmd if cmd is None MSG.RUN_FUNCTION + # self.data = data - def isDone( self ): - return self.isSet() + # def isDone( self ): + # return self.isSet() - def join( self ): - self.wait( ) + # def join( self ): + # self.wait( ) class Worker( threading.Thread ): def __init__( self, console, queue, files_to_ignore ): @@ -257,6 +314,8 @@ class Worker( threading.Thread ): self.queue.task_done( ) def main( args ): + start = time.clock() + # check requirements if call_process( 'p4 -V' ) != 0: print( 'Perforce Command-line Client(p4) is required for this script.' ) @@ -269,11 +328,60 @@ def main( args ): parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 ) parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False ) parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=True ) + parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False ) - ( options, args ) = parser.parse_args( ) + ( options, args ) = parser.parse_args( args ) directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) + # get user + print("\nChecking p4 info...") + result = get_p4_py_results('info') + if len(result) == 0 or b'userName' not in result[0].keys(): + print("Can't find perforce info, is it even setup?") + sys.exit(1) + username = get_str_from_process_stdout(result[0][b'userName']) + client_host = get_str_from_process_stdout(result[0][b'clientHost']) + print("|Done.") + + client_root = get_client_root() + ldirectory = directory.lower() + workspace_name = None + if not ldirectory.startswith(client_root.lower()): + print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...") + + workspace_name = parse_info_from_command('p4 info', 'Client name: ') + + # get user workspaces + result = get_p4_py_results('workspaces -u ' + username) + workspaces = [] + for r in result: + whost = get_str_from_process_stdout(r[b'Host']) + if whost is not None and len(whost) != 0 and client_host != whost: + continue + workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])} + workspaces.append(workspace) + + del result + + # check current directory against current workspace, see if it matches existing workspaces. + for w in workspaces: + wname = w['name'] + wlower = w['root'].lower() + if ldirectory.startswith(wlower): + # set current directory, don't forget to revert it back to the existing one + print("|Setting client view to: " + wname) + + if try_call_process( 'p4 set P4CLIENT=' + wname ): + print("|There was a problem trying to set the p4 client view (workspace).") + sys.exit(1) + break + else: + print( "|Couldn't find a workspace root that matches the current directory for the current user." ) + sys.exit(1) + print("|Done.") + + # Files are added from .p4ignore # Key is the file root, the value is the table of file regexes for that directory. files_to_ignore = PDict() @@ -285,11 +393,13 @@ def main( args ): with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: if not options.quiet: - c.writeflush( "Caching files in depot, this may take a little while..." ) + c.writeflush( "\nCaching files in depot, this may take a little while..." ) # TODO: push this off to a thread and walk the directory so we get a headstart. files_in_depot = get_client_set( directory ) + c.writeflush( "|Done." ) + # TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory? # TODO: at this point join on both tasks to wait until they're done @@ -299,7 +409,7 @@ def main( args ): # will at least need to redo how the ignore lists are handled for efficiencies sake. if not options.quiet: - c.writeflush( "Checking " + directory) + c.writeflush( "\nChecking " + directory) for root, dirs, files in os.walk( directory ): ignore_list = get_ignore_list( root, files_to_ignore ) @@ -346,21 +456,35 @@ def main( args ): if match_in_ignore_list( path, ignore_list ): # add option of using send2trash if not options.quiet: - c.write( "| ignoring " + d ) + c.write( "| ignoring " + path ) dirs.remove( d ) try: os.rmdir(path) remove_dir_count += 1 if not options.quiet: - c.write( "| " + d + " was removed." ) + c.write( "| " + path + " was removed." ) except OSError: # Fails on non-empty directory pass if not options.quiet: c.write( "|Done." ) + # This needs to happen automatically even when an exception happens, when we leave scope. + if workspace_name is not None: + c.write("\nReverting back to original client view...") + # set workspace back to the original one + if try_call_process( 'p4 set P4CLIENT=' + workspace_name ): + error_count += 1 + if not options.quiet: + c.write("|There was a problem trying to restore the set p4 client view (workspace).") + else: + if not options.quiet: + c.write("|Reverted client view back to '" + workspace_name + "'.") + if not options.quiet: + c.write("|Done.") + if not options.quiet: - output = "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " ) + output = "\n[ Removed " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " ) output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories") if warning_count > 0: @@ -368,7 +492,11 @@ def main( args ): if error_count > 0: output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" ) - c.write( output + "." ) + end = time.clock() + delta = end - start + output += ", and finished in " + str(delta) + "s" + + c.write( output + " ]" ) if __name__ == "__main__": try: From e7bb65874e29038fbd9d57c77c6d632686b5def6 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 22 Oct 2014 12:05:57 -0600 Subject: [PATCH 16/32] Added more debug info at the end for files processed, cleaned up formatting. At some point will strive to make the output more UNIX friendly, parsable. I also fixed a bug where the script would crash instead of setting the P4Client. I need to fix the script to use a `with` construct so if you terminate the program the P4Client is returned to what it was. --- README.md | 5 ++++- p4RemoveUnversioned.py | 31 +++++++++++++++++++++---------- python2.7.exe.stackdump | 16 ---------------- 3 files changed, 25 insertions(+), 27 deletions(-) delete mode 100644 python2.7.exe.stackdump diff --git a/README.md b/README.md index abd326d..952f42b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ p4RemoveUnversioned =================== -Removes unversioned files from perforce repository. Script is in beta, though it works. It's a little slow due to the way fstat is used, but that will be changed soon, so the speed up should be enormous once that's done, up to 100x or more. +Removes unversioned files from perforce repository. Script is in beta, works well, but still going through continued testing. There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required. + +Concerning benchmarks: I used to have a HDD, now a SSD. So I can't provide valid comparisons to the old numbers until I do them on a computer with a HDD. That said, this single worker implementation runs faster than the old multi-threaded version. Can't wait to further update it, will only continue to get faster. + This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex. diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 28bae51..06c8d8f 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -237,8 +237,8 @@ class Console( threading.Thread ): self.buffer_write_times.clear( ) self.queue.task_done( ) - print(self.queue.qsize()) - print(self.queue.empty()) + #print(self.queue.qsize()) + #print(self.queue.empty()) break elif event == Console.MSG.WRITE: @@ -347,7 +347,7 @@ def main( args ): client_root = get_client_root() ldirectory = directory.lower() workspace_name = None - if not ldirectory.startswith(client_root.lower()): + if client_root is None or not ldirectory.startswith(client_root.lower()): print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...") workspace_name = parse_info_from_command('p4 info', 'Client name: ') @@ -386,6 +386,9 @@ def main( args ): # Key is the file root, the value is the table of file regexes for that directory. files_to_ignore = PDict() + processed_file_count = 0 + processed_directory_count = 0 + remove_file_count = 0 remove_dir_count = 0 warning_count = 0 @@ -414,18 +417,21 @@ def main( args ): ignore_list = get_ignore_list( root, files_to_ignore ) if not options.quiet: - c.write( "|Checking " + root ) + c.write( "|Checking " + os.path.relpath( root, directory ) ) for d in dirs: + processed_directory_count += 1 path = join( root, d ) + rel_path = os.path.relpath( path, directory ) if match_in_ignore_list( path, ignore_list ): # add option of using send2trash if not options.quiet: - c.write( "| ignoring " + d ) + c.write( "| ignoring " + rel_path ) dirs.remove( d ) for f in files: + processed_file_count += 1 path = normpath( join( root, f ) ) if path not in files_in_depot: @@ -451,18 +457,20 @@ def main( args ): ignore_list = get_ignore_list( root, files_to_ignore ) for d in dirs: + processed_directory_count += 1 path = os.path.join( root, d ) + rel_path = os.path.relpath( path, directory ) if match_in_ignore_list( path, ignore_list ): # add option of using send2trash if not options.quiet: - c.write( "| ignoring " + path ) + c.write( "| ignoring " + rel_path ) dirs.remove( d ) try: os.rmdir(path) remove_dir_count += 1 if not options.quiet: - c.write( "| " + path + " was removed." ) + c.write( "| " + rel_path + " was removed." ) except OSError: # Fails on non-empty directory pass @@ -484,7 +492,10 @@ def main( args ): c.write("|Done.") if not options.quiet: - output = "\n[ Removed " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " ) + output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " ) + output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories") + + output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " ) output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories") if warning_count > 0: @@ -494,9 +505,9 @@ def main( args ): end = time.clock() delta = end - start - output += ", and finished in " + str(delta) + "s" + output += "\nFinished in " + str(delta) + "s" - c.write( output + " ]" ) + c.write( output ) if __name__ == "__main__": try: diff --git a/python2.7.exe.stackdump b/python2.7.exe.stackdump deleted file mode 100644 index 2050f78..0000000 --- a/python2.7.exe.stackdump +++ /dev/null @@ -1,16 +0,0 @@ -Stack trace: -Frame Function Args -000FD4FC6C0 0018006F733 (FFFFFFFFFFFFFFFF, 006002C2510, 005CCDD1850, 005CCDD1870) -00000000006 00180070C6A (000FD4FC8A0, 00000000000, 0000000035C, 00000000000) -000FD4FC8A0 00180118778 (00000000000, 00000000000, 000FD4FCA20, 00600000003) -00000000041 0018011587E (00180133B2D, 000000002CC, 00000000000, 00000000000) -00000000000 00180115D4B (001801362C0, 006002C2270, 000775D19A1, 00000000006) -00000000000 00180115F1C (00000000000, 0008FB10544, 7FEFD6464DA, 006002C2318) -00000000000 001801161DF (0018013B2FB, 00600000001, 001801CA2C0, 000FD4FD640) -00000000000 00180144816 (00180135C7F, 006002C2270, 006002C2270, 006002C2270) -00000000000 001800BFCE3 (0018013460D, 00000000000, 005CCDB7127, 006002C2318) -00000000000 001801629FC (00000000000, 00000000000, 00000000000, 006002C2270) -00000000000 00180136AEF (00000000000, 00000000000, 00000000000, 00000000000) -00000000000 00180136334 (000003A0000, 00000000000, 00000000000, 00000000000) -00000000000 001800C373B (000003A0000, 00000000000, 00000000000, 00000000000) -End of stack trace From 9d4d26250df72ec8b5652852f3a95bc1607a6a98 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 14 Jan 2015 17:58:34 -0700 Subject: [PATCH 17/32] Added a fix if the specified directory isn't added to the repo but still in one, it'll be scanned and contents cleaned up. The only thing is, as of right now the folder itself won't be deleted, you'd have to run the script from a higher directory. --- p4RemoveUnversioned.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 06c8d8f..12ae84a 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -152,6 +152,8 @@ def get_client_set( path ): proc.wait( ) for line in proc.stderr: + if "no such file" in line: + continue raise Exception(line) return files From 1d1d7f8cae7dedc119a4566d6c61c7d66423d1d2 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 18 Feb 2015 15:09:57 -0700 Subject: [PATCH 18/32] Split scripts up for now, may add all in one scripts later. Added p4SyncMissingFiles.py, so you don't have to do a force sync and redownload everything. --- README.md | 4 +- p4Helper.py | 398 ++++++++++++++++++++++++++++ p4Helper.pyc | Bin 0 -> 13085 bytes p4RemoveUnversioned.py | 586 ++++++++--------------------------------- p4SyncMissingFiles.py | 136 ++++++++++ 5 files changed, 648 insertions(+), 476 deletions(-) create mode 100644 p4Helper.py create mode 100644 p4Helper.pyc create mode 100644 p4SyncMissingFiles.py diff --git a/README.md b/README.md index 952f42b..5942e43 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -p4RemoveUnversioned +p4Tools =================== Removes unversioned files from perforce repository. Script is in beta, works well, but still going through continued testing. There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required. @@ -6,6 +6,6 @@ Removes unversioned files from perforce repository. Script is in beta, works wel Concerning benchmarks: I used to have a HDD, now a SSD. So I can't provide valid comparisons to the old numbers until I do them on a computer with a HDD. That said, this single worker implementation runs faster than the old multi-threaded version. Can't wait to further update it, will only continue to get faster. -This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex. +~~This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex.~~ I need to re-add this to the newer script. **Files are currently permanently deleted, so use this at your own risk.** diff --git a/p4Helper.py b/p4Helper.py new file mode 100644 index 0000000..d50bd63 --- /dev/null +++ b/p4Helper.py @@ -0,0 +1,398 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- +# author : Brian Ernst +# python_version : 2.7.6 and 3.4.0 +# ================================= + +import datetime, inspect, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading + +# trying ntpath, need to test on linux +import ntpath + + +try: input = raw_input +except: pass + + +#============================================================== +re_remove_comment = re.compile( "#.*$" ) +def remove_comment( s ): + return re.sub( re_remove_comment, "", s ) + +def singular_pulural( val, singular, plural ): + return singular if val == 1 else plural + + +#============================================================== +def enum(*sequential, **named): + enums = dict(zip(sequential, range(len(sequential))), **named) + return type('Enum', (), enums) + +MSG = enum('SHUTDOWN', 'PARSE_DIRECTORY', 'RUN_FUNCTION') + +p4_ignore = ".p4ignore" + +main_pid = os.getpid( ) + + +#============================================================== +#if os.name == 'nt' or sys.platform == 'cygwin' +def basename( path ): + # TODO: import based on platform + # https://docs.python.org/2/library/os.path.html + # posixpath for UNIX-style paths + # ntpath for Windows paths + # macpath for old-style MacOS paths + # os2emxpath for OS/2 EMX paths + + #return os.path.basename( path ) + return ntpath.basename( path ) + +def normpath( path ): + return ntpath.normpath( path ) + +def join( patha, pathb ): + return ntpath.join( patha, pathb ) + +def splitdrive( path ): + return ntpath.splitdrive( path ) + +#============================================================== +def get_ignore_list( path, files_to_ignore ): + # have to split path and test top directory + dirs = path.split( os.sep ) + + ignore_list = [ ] + + for i, val in enumerate( dirs ): + path_to_find = os.sep.join( dirs[ : i + 1] ) + + if path_to_find in files_to_ignore: + ignore_list.extend( files_to_ignore[ path_to_find ] ) + + return ignore_list + +def match_in_ignore_list( path, ignore_list ): + for r in ignore_list: + if re.match( r, path ): + return True + return False + + +#============================================================== +def call_process( args ): + return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + +def try_call_process( args, path=None ): + try: + subprocess.check_output( args.split( ), shell=False, cwd=path ) + return 0 + except subprocess.CalledProcessError: + return 1 + +use_bytearray_str_conversion = type( b"str" ) is not str +def get_str_from_process_stdout( line ): + if use_bytearray_str_conversion: + return ''.join( map( chr, line ) ) + else: + return line + +def parse_info_from_command( args, value, path = None ): + """ + + :rtype : string + """ + proc = subprocess.Popen( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) + for line in proc.stdout: + line = get_str_from_process_stdout( line ) + + if not line.startswith( value ): + continue + return line[ len( value ) : ].strip( ) + return None + +def get_p4_py_results( args, path = None ): + results = [] + proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) + try: + while True: + output = marshal.load( proc.stdout ) + results.append( output ) + except EOFError: + pass + finally: + proc.stdout.close() + return results + + +#============================================================== +def fail_if_no_p4(): + if call_process( 'p4 -V' ) != 0: + print( 'Perforce Command-line Client(p4) is required for this script.' ) + sys.exit( 1 ) + +# Keep these in mind if you have issues: +# https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime +# https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running +def get_client_set( path ): + files = set( [ ] ) + + make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False + + command = "p4 fstat ..." + + proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path ) + for line in proc.stdout: + line = get_str_from_process_stdout( line ) + + clientFile_tag = "... clientFile " + if not line.startswith( clientFile_tag ): + continue + + local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) ) + if make_drive_upper: + drive, path = splitdrive( local_path ) + local_path = ''.join( [ drive.upper( ), path ] ) + + files.add( local_path ) + + proc.wait( ) + + for line in proc.stderr: + if "no such file" in line: + continue + raise Exception(line) + + return files + +def get_client_root( ): + """ + + :rtype : string + """ + command = "p4 info" + + proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + for line in proc.stdout: + line = get_str_from_process_stdout( line ) + + clientFile_tag = "Client root: " + if not line.startswith( clientFile_tag ): + continue + + local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) ) + + return local_path + return None + +class P4Workspace: + """ + Use this class when working in a workspace. + Makes sure the environmentals are setup correctly, and that you aren't working on a non-perforce directory accidentally; + otherwise you can delete files that shouldn't be deleted. Ex: + + with P4Workspace( cwd ): #sets current workspace to cwd, or fails + # do stuff here + # on exit reverts to previous set workspace + """ + + def __init__( self, directory): + self.directory = directory + + def __enter__( self ): + # get user + #print("\nChecking p4 info...") + result = get_p4_py_results('info') + if len(result) == 0 or b'userName' not in result[0].keys(): + print("Can't find perforce info, is it even setup?") + sys.exit(1) + username = get_str_from_process_stdout(result[0][b'userName']) + client_host = get_str_from_process_stdout(result[0][b'clientHost']) + #print("|Done.") + + # see if current directory is set to current workspace, if not, set it to current workspace. + client_root = get_client_root() + ldirectory = self.directory.lower() + oldworkspace_name = None + + if client_root is None or not ldirectory.startswith(client_root.lower()): + #print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...") + + oldworkspace_name = parse_info_from_command('p4 info', 'Client name: ') + + # get user workspaces + result = get_p4_py_results('workspaces -u ' + username) + workspaces = [] + for r in result: + whost = get_str_from_process_stdout(r[b'Host']) + if whost is not None and len(whost) != 0 and client_host != whost: + continue + workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])} + workspaces.append(workspace) + + del result + + # check current directory against current workspace, see if it matches existing workspaces. + for w in workspaces: + wname = w['name'] + wlower = w['root'].lower() + if ldirectory.startswith(wlower): + # set current directory, don't forget to revert it back to the existing one + #print("|Setting client view to: " + wname) + + if try_call_process( 'p4 set P4CLIENT=' + wname ): + #print("|There was a problem trying to set the p4 client view (workspace).") + sys.exit(1) + break + else: + # TODO: look up workspace/users for this computer + print( "Couldn't find a workspace root that matches the current directory for the current user." ) + sys.exit(1) + #print("|Done.") + self.oldworkspace_name = oldworkspace_name + return self + + def __exit__( self, type, value, tb ): + # If we changed the current workspace, switch it back. + if self.oldworkspace_name is not None: + c.write("\nReverting back to original client view...") + # set workspace back to the original one + if try_call_process( 'p4 set P4CLIENT=' + self.oldworkspace_name ): + # error_count += 1 # have console log errors + # if not options.quiet: + print("There was a problem trying to restore the set p4 client view (workspace).") + sys.exit(1) + #else: + # if not options.quiet: + # c.write("|Reverted client view back to '" + self.oldworkspace_name + "'.") + #if not options.quiet: + # c.write("|Done.") + +#============================================================== +class PTable( list ): + def __init__( self, *args ): + list.__init__( self, args ) + self.mutex = multiprocessing.Semaphore( ) + +class PDict( dict ): + def __init__( self, *args ): + dict.__init__( self, args ) + self.mutex = multiprocessing.Semaphore( ) + + +#============================================================== +# TODO: Create a child thread for triggering autoflush events +# TODO: Hook console into stdout so it catches print +class Console( threading.Thread ): + MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' ) + + # auto_flush_time is time in milliseconds since last flush to trigger a flush when writing + def __init__( self, auto_flush_num = None, auto_flush_time = None ): + threading.Thread.__init__( self ) + self.buffers = {} + self.buffer_write_times = {} + self.running = True + self.queue = multiprocessing.JoinableQueue( ) + self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1 + self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1 + self.shutting_down = False + + def write( self, data, pid = None ): + self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) ) + + def writeflush( self, data, pid = None ): + pid = pid if pid is not None else os.getpid() + self.queue.put( ( Console.MSG.WRITE, pid, data ) ) + self.queue.put( ( Console.MSG.FLUSH, pid ) ) + + def flush( self, pid = None ): + self.queue.put( ( Console.MSG.FLUSH, pid if pid is not None else os.getpid() ) ) + + def clear( self, pid = None ): + self.queue.put( ( Console.MSG.CLEAR, pid if pid is not None else os.getpid() ) ) + + def __enter__( self ): + self.start( ) + return self + + def __exit__( self, type, value, tb ): + self.queue.put( ( Console.MSG.SHUTDOWN, ) ) + self.queue.join( ) + + def run( self ): + while True: + data = self.queue.get( ) + event = data[0] + + if event == Console.MSG.SHUTDOWN: + # flush remaining buffers before shutting down + for ( pid, buffer ) in self.buffers.items( ): + for line in buffer: + print( line ) + self.buffers.clear( ) + self.buffer_write_times.clear( ) + self.queue.task_done( ) + + #print(self.queue.qsize()) + #print(self.queue.empty()) + break + + elif event == Console.MSG.WRITE: + pid, s = data[ 1 : ] + + if pid not in self.buffers: + self.buffers[ pid ] = [] + if pid not in self.buffer_write_times: + self.buffer_write_times[ pid ] = datetime.datetime.now( ) + self.buffers[ pid ].append( s ) + + if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num: + self.flush( pid ) + elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time: + self.flush( pid ) + # TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time + elif event == Console.MSG.FLUSH: + pid = data[ 1 ] + if pid in self.buffers: + for line in self.buffers[ pid ]: + print( line ) + self.buffers.pop( pid, None ) + self.buffer_write_times[ pid ] = datetime.datetime.now( ) + elif event == Console.MSG.CLEAR: + pid = data[ 1 ] + if pid in self.buffers: + self.buffers.pop( pid, None ) + + self.queue.task_done( ) + + +#============================================================== +# class Task( threading.Event ): +# def __init__( data, cmd = None ): +# threading.Event.__init__( self ) + +# self.cmd = cmd if cmd is None MSG.RUN_FUNCTION +# self.data = data + +# def isDone( self ): +# return self.isSet() + +# def join( self ): +# self.wait( ) + + +#============================================================== +class Worker( threading.Thread ): + def __init__( self, console, queue, commands ): + threading.Thread.__init__( self ) + + self.queue = queue + self.commands = commands + + def run( self ): + while True: + ( cmd, data ) = self.queue.get( ) + if not self.commands[cmd](data): + self.queue.task_done( ) + break + self.queue.task_done( ) diff --git a/p4Helper.pyc b/p4Helper.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4884754adb315b7046b77cc97aafc988f1eac1b GIT binary patch literal 13085 zcmcgyO>i8?b?%v6{4B5_Kma5NiWbKtC4m$P%4JA$Xj!HNfIqghga)8jMv)p1X9mOo zyEBXFSpskXmnE_sIkBC;N}S5CQmLeJNK%zUPR>15IpmN-ZmyhCm2+~B^L?-PPb8d# zgcXR}dehTCuV26Sz1Od&@Sh_?b@gv6folB~@&Bv%X=N)F;-95zO09NiORZY%dRwj9 zZMvXV3vIfnR*P-APp$T~>3+4^-=+uD>Oh+wRI7t+x};W1ZF)$p4z=lFwK}YJhg4Kn zt0StK3@FR85=*@w^{b5|Dzw!@ur{i~f_emg$5dDp;;0Jygcw)RxC${prX5rd!S93$ zG25hy`qaiT^$@*|t8hpfPpNQNh!ZL-3o)(25g|^h@Q4u4sc=+?Qz{%2;&YT%cvL+C zC#O~9sBoNIMW0ujnCOfOC#2a~X*Ma%&Ix@?=oeIYTpGNn!YLtMQsD_9W>g3U(D5W6 z!Rj|x&#Uk`6EoR_ooa9BWRD3Ge)g;FyGHsNy? z<)n4-8wfXE<`sLeHR3Qs|dKUAuB;DN8q^AlG*qbFTEdojiy# z9j8eYqV7giYed;>V>ds7_MaY?JZdtatbh#g$H#g6^dt%}4c0epm5r%AEPc;zuOKEj zh8gw}{vY_Ye2`t^q zC-=?1)v!fulJ7Pmv#CRP($Q8EFJcIV%@e;Kg+fIb{4o2-sW`@SoP-a`I%kRCmv7u& zS-4fLud>wItMp4NVUp@P#yC*ONyPHieJso%M6j>ic zftA&T6KbnFjp7hIokaI9;@4I@N{T-XS=oM!be!n?ugc-~VY(4SS-wjarW;1Ui)z2G)pVms>)jJeo^oL0`~7MU{{UM3 zDtkfg71Tx_A5l4t*Vfhx3jeC8Z$YsEGJ=O}f#%u;lv8a$Icnsz6k1%A`FZ40%3=V~ zP97!U4A&wu60=*2YmxTy)H8O~!j4Sm!)RveyjrYtf%DjviD{bLdv;rkF(jU!5#(w4 zfTh;#GSgR3r3wDZ9=H4LY3rn2woceVTX(6EvJZ49JLf2^g}bK&A&vP%z`##9fdRtl z^45T=LNsmFW_OobCi1KMd9W^vUdft~R9*9H+Tfx{$$gP+*&#c`Iw#Rz=JMjCcd=hb zy&goNtPyM68nimlK}pLpUjWfDQ>vRdIo2RngIf^dp=Gxn0z2wFOq;pvI>;!?X1MHJ ztfhIck)=VTwJbaEYc=;YU$Vv0&85XIDEis`!vH_kgu{ES`IhnAHg#?$f1Oowsk<*7 zv`|?k;mNWs1cuAiAq_Q^|Lxx4Q{C=pg$+XgnF^%=8Ruh{Dw!;|J%GeHD zxL-gaE+$xyf=v&?YBY0^gL{#$#!<|J_b5cXLSZdtS(*t5x^o=haNwK^_GY_YuYA6N zaW%I{Sq1AfbYaXgCY@&;t>7k!R-6L?A!LYK0YVbBp|MeX0LL<@Wf2IaTyaq-qXVZv zHFf0O+sz|C%lutW=NTfmb`)mtqJUO)zhT6;o^^Gi7AMgm?0{fId*_^MSz2%F1}dnr zAKe)O zu+7LoIrV~O=6)5#A@+x%NW;&dA91nm6^?Txb) zcE;?ZwRKHZtI1JdLP-GOVf#>71;?h>#t^sBdLLS`JVW)Wh|6ru2^?Lz{@Aw7(D%u% zxo?poJi4-BbUDxt+D4|=5h_#ewbTz~1i@Rsfjtf-oLjtgO<+SdE2yP#D*`@q9FG7Z zLuHX})^hD$V+ENrV04I0u3Im9{#&<%RFKM=qxlGtKSFmN)Twy zo%#YfglD3>5P`fxW0E?$8LT@DYiD>WGYS*suEPhKD-V9|dwk-SHfnwjovXW4ck%rP8ouEA zRW^B>1-*cPv`dw6d1Nqtlj`3D0%S6;ABHmJwjbx>6(lrY+zAk*BjGVay)r<+eiDjH zWKj1vBTr5=o*9gg(}H2a34wXZkaVTs<^Fx~BefJ!g~tG0fXAGSFu*g0=tt0#Xkupa zbd-M|H6}2G7r1~J-S~rk3m(Bf1$Y^^p0|(MLpHnwp2qBevvY}rH%d%%2$&4ij_}{> z&D#+zjKz2c#C@=`o!g><^y6O>*8+cG)Dw1v6PCDq<0fbbVauDtqR>*kGOH@hHO!f8KFNs5MZJ^y}$?0!tu9{|>y5R?rY#^rx_909bX<~ z-dRUFybF7PM`&j37dcV#AkNZ+sYaaD93R!t%4Wj}(hRAgytaGMLBN8xK#<+E$ri~= zd1sE4b0leU33h5cQwZM^d!h@5Pc!MC{n2y5ZxP~MLh#cdx_ZS<~y#QS`17E zsjC0TX2aZXOl<~QQ9YTh{#lM0R zpaBqrrYkyH#}5Jk!HN(BPKl=HI!wSXuds z=9~F8I0)Z8@FqUPAc!P~0CL2)HXd}Wt`!w`-%uvAGemWFi3J%K#x|o}Ey0b;n8>}# z0&1$nQlSbwt!zh`yTCdExqF?(*I8UaAs0GnE$m3alRP=cccHN6fMQM*5nn4uL93RNguGYI|1tVwIKSdysbOkuPzhFE3 zG;nNX`=E6JqYk+O`1G6i{9u!~pJws>IPq((@Ys0}N85;&4_3~b7-NNg#M$=YN8vQm zd$p+UFA#BkVZxJBc3so2nOv#T<&Gt=3i^R1KG0EU#^6%~_h3(U4(B$)$$B%7c1$XikRbV}u3(;8CY10|tfh4~IA+P6qzjgniv=rT ztvmj+3I2N+XpECkVEvY?V`%|*qye7gkJ2O{>d#i(G zZ@81~7xV8dzU4moLBWT7FNae^lt5>qaBS?TpjrfSHwf8)76|X!5h5QEy2-(aX#Wmr z8B*7RibrBOQ^)=k}5{cexMy>lWd=`K# z^8y<%4Dg%0^{X}Yx`!)9fsCFi4sUKyIGA3?B@~7SmTo7;tOYTCjCbzuq3D|RGr`6g zDj>ry1vgxh|3 zH7F=HUG@PpA)P;n(*0c&lHTE!(GS`W?=D|=f5_LAl*u-6V{qzj#9?>sg1DiN+!lA1 zukgQTJJq}2MbC?E8&n4@`YsxDwGGKVvqW=iEg5=>L?i({M)m|tN1N`_AAgNJimlcp z@6s!l66Wcy+xd|3tfQ4_UOWsEAA-5<4fe~9uq|mvLAAt1XNk#T+gomzb^0B zKkMZMHQY+R-m^TAl6B($#gYj^X7^yF7N~YW!xnI?W0007c20;yctio%K-%&eX6d|<1bK&k8 z`Sb-B9=OApU}|LH?l6-l$RAw4u~k*otp(g6U;=cJ+=9re)cb&`&Ar`H@)N5&<*=F3 zB0IJ^eFw=EOd}VBW;~{T{)P85<`HRGY}WT?;w-&d|&s@nN*}jx?@TjOW-|5*gx?Rnv9JK5*LU6S#6= z$8j0Nbm|=>`ALt+CuRL*+pK?@nQybSez|j5vUg7Qu=gpk{}ZlWWO=x`K_@7r91#u_ z1$a*udZwL{X~CO_{eWo?7@1=j@pkrdtDvf42voTW!l?FV8N949moOdw3g6k|Jj679 z3$4V)0rqw6{wX{CHtXnua$j#E)JP(6G;+j)r;Qksh__ABZIkJc_=7m&e!_y0h
36Y!S=sFyRMj?T0)!<7pCo4Mv_Yu
z0d=zDeyVubS>nPnSIoW**u+IiZPPBXKp;0(3$)U2x7#7J1dUqiny9Jhm?$T!m^-PL
zL7GMV87Z#J{c{xEKu&ZusD~2J9b(ngt&h3tG6?GXF=+3IRkVzx!EsMo;1{Bae~lqO
zXST1Mi1^Hm`S;nlUdMt+aY
zMTe%?hbWc#+qm+;*}>qLQ1rpjs0MDWU2d<-NY7%m@M3zl!(5H;t>F$poZ|lO0hU= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num:
-                    self.flush( pid )
-                elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time:
-                    self.flush( pid )
-                # TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time
-            elif event == Console.MSG.FLUSH:
-                pid = data[ 1 ]
-                if pid in self.buffers:
-                    for line in self.buffers[ pid ]:
-                        print( line )
-                    self.buffers.pop( pid, None )
-                    self.buffer_write_times[ pid ] = datetime.datetime.now( )
-            elif event == Console.MSG.CLEAR:
-                pid = data[ 1 ]
-                if pid in self.buffers:
-                    self.buffers.pop( pid, None )
-
-            self.queue.task_done( )
-
-# class Task( threading.Event ):
-    # def __init__( data, cmd = None ):
-        # threading.Event.__init__( self )
-
-        # self.cmd = cmd if cmd is None MSG.RUN_FUNCTION
-        # self.data = data
-
-    # def isDone( self ):
-        # return self.isSet()
-
-    # def join( self ):
-        # self.wait( )
-
-class Worker( threading.Thread ):
-    def __init__( self, console, queue, files_to_ignore ):
-        threading.Thread.__init__( self )
-
-        self.console = console
-        self.queue = queue
-        self.files_to_ignore = files_to_ignore
-
-    def run( self ):
-        while True:
-            ( cmd, data ) = self.queue.get( )
-
-            if cmd == MSG.SHUTDOWN:
-                self.console.flush( )
-                self.queue.task_done( )
-                break
-
-            if cmd == MSG.RUN_FUNCTION:
-                break
-
-            if cmd != MSG.PARSE_DIRECTORY or data is None:
-                self.console.flush( )
-                self.queue.task_done( )
-                continue
-
-            directory = data
-
-            # add threading stuffs
-
-            self.queue.task_done( )
-
+#==============================================================
 def main( args ):
     start = time.clock()
 
-    # check requirements
-    if call_process( 'p4 -V' ) != 0:
-        print( 'Perforce Command-line Client(p4) is required for this script.' )
-        sys.exit( 1 )
+    fail_if_no_p4()
 
     #http://docs.python.org/library/optparse.html
     parser = optparse.OptionParser( )
@@ -336,184 +35,123 @@ def main( args ):
 
     directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
 
-    # get user
-    print("\nChecking p4 info...")
-    result = get_p4_py_results('info')
-    if len(result) == 0 or b'userName' not in result[0].keys():
-        print("Can't find perforce info, is it even setup?")
-        sys.exit(1)
-    username = get_str_from_process_stdout(result[0][b'userName'])
-    client_host = get_str_from_process_stdout(result[0][b'clientHost'])
-    print("|Done.")
-
-    client_root = get_client_root()
-    ldirectory = directory.lower()
-    workspace_name = None
-    if client_root is None or not ldirectory.startswith(client_root.lower()):
-        print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...")
-
-        workspace_name = parse_info_from_command('p4 info', 'Client name: ')
-
-        # get user workspaces
-        result = get_p4_py_results('workspaces -u ' + username)
-        workspaces = []
-        for r in result:
-            whost = get_str_from_process_stdout(r[b'Host'])
-            if whost is not None and len(whost) != 0 and client_host != whost:
-                continue
-            workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])}
-            workspaces.append(workspace)
-
-        del result
-
-        # check current directory against current workspace, see if it matches existing workspaces.
-        for w in workspaces:
-            wname = w['name']
-            wlower = w['root'].lower()
-            if ldirectory.startswith(wlower):
-                # set current directory, don't forget to revert it back to the existing one
-                print("|Setting client view to: " + wname)
-
-                if try_call_process( 'p4 set P4CLIENT=' + wname ):
-                    print("|There was a problem trying to set the p4 client view (workspace).")
-                    sys.exit(1)
-                break
-        else:
-            print( "|Couldn't find a workspace root that matches the current directory for the current user." )
-            sys.exit(1)
-        print("|Done.")
-
-
-    # Files are added from .p4ignore
-    # Key is the file root, the value is the table of file regexes for that directory.
-    files_to_ignore = PDict()
-
-    processed_file_count = 0
-    processed_directory_count = 0
-    
-    remove_file_count = 0
-    remove_dir_count = 0
-    warning_count = 0
-    error_count = 0
-
     with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
-        if not options.quiet:
-            c.writeflush( "\nCaching files in depot, this may take a little while..." )
+        with P4Workspace( directory ):
+            # Files are added from .p4ignore
+            # Key is the file root, the value is the table of file regexes for that directory.
+            files_to_ignore = PDict()
 
-        # TODO: push this off to a thread and walk the directory so we get a headstart.
-        files_in_depot = get_client_set( directory )
-
-        c.writeflush( "|Done." )
-
-        # TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?
-
-        # TODO: at this point join on both tasks to wait until they're done
-
-        # TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
-        #       need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
-        #       will at least need to redo how the ignore lists are handled for efficiencies sake.
-
-        if not options.quiet:
-            c.writeflush( "\nChecking " + directory)
-        for root, dirs, files in os.walk( directory ):
-            ignore_list = get_ignore_list( root, files_to_ignore )
-
-            if not options.quiet:
-                c.write( "|Checking " + os.path.relpath( root, directory ) )
-
-            for d in dirs:
-                processed_directory_count += 1
-                path = join( root, d )
-                rel_path = os.path.relpath( path, directory )
-
-                if match_in_ignore_list( path, ignore_list ):
-                    # add option of using send2trash
-                    if not options.quiet:
-                        c.write( "| ignoring " + rel_path )
-                    dirs.remove( d )
-
-            for f in files:
-                processed_file_count += 1
-                path = normpath( join( root, f ) )
-
-                if path not in files_in_depot:
-                    if not options.quiet:
-                        c.write( "| " + f + " is unversioned, removing it." )
-                    try:
-                        os.chmod( path, stat.S_IWRITE )
-                        os.remove( path )
-                        remove_file_count += 1
-                    except OSError as ex:
-                        c.writeflush( "|  " + type( ex ).__name__ )
-                        c.writeflush( "|  " + repr( ex ) )
-                        c.writeflush( "|  ^ERROR^" )
-
-                        error_count += 1
-        if not options.quiet:
-            c.write( "|Done." )
-
-        if not options.quiet:
-            c.write( os.linesep + "Removing empty directories...")
-        # remove empty directories in reverse order
-        for root, dirs, files in os.walk( directory, topdown=False ):
-            ignore_list = get_ignore_list( root, files_to_ignore )
-
-            for d in dirs:
-                processed_directory_count += 1
-                path = os.path.join( root, d )
-                rel_path = os.path.relpath( path, directory )
-
-                if match_in_ignore_list( path, ignore_list ):
-                    # add option of using send2trash
-                    if not options.quiet:
-                        c.write( "| ignoring " + rel_path )
-                    dirs.remove( d )
-                try:
-                    os.rmdir(path)
-                    remove_dir_count += 1
-                    if not options.quiet:
-                        c.write( "| " + rel_path + " was removed." )
-                except OSError:
-                    # Fails on non-empty directory
-                    pass
-        if not options.quiet:
-            c.write( "|Done." )
-
-        # This needs to happen automatically even when an exception happens, when we leave scope.
-        if workspace_name is not None:
-            c.write("\nReverting back to original client view...")
-            # set workspace back to the original one
-            if try_call_process( 'p4 set P4CLIENT=' + workspace_name ):
-                error_count += 1
-                if not options.quiet:
-                    c.write("|There was a problem trying to restore the set p4 client view (workspace).")
-            else:
-                if not options.quiet:
-                    c.write("|Reverted client view back to '" + workspace_name + "'.")
-            if not options.quiet:
-                c.write("|Done.")
-
-        if not options.quiet:
-            output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " )
-            output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories")
+            processed_file_count = 0
+            processed_directory_count = 0
             
-            output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " )
-            output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories")
+            remove_file_count = 0
+            remove_dir_count = 0
+            warning_count = 0
+            error_count = 0
 
-            if warning_count > 0:
-                output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" )
-            if error_count > 0:
-                output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )
+            if not options.quiet:
+                c.writeflush( "\nCaching files in depot, this may take a little while..." )
 
-            end = time.clock()
-            delta = end - start
-            output += "\nFinished in " + str(delta) + "s"
+            # TODO: push this off to a thread and walk the directory so we get a headstart.
+            files_in_depot = get_client_set( directory )
 
-            c.write( output )
+            c.writeflush( "|Done." )
+
+            # TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?
+
+            # TODO: at this point join on both tasks to wait until they're done
+
+            # TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
+            #       need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
+            #       will at least need to redo how the ignore lists are handled for efficiencies sake.
+
+            if not options.quiet:
+                c.writeflush( "\nChecking " + directory)
+            for root, dirs, files in os.walk( directory ):
+                ignore_list = get_ignore_list( root, files_to_ignore )
+
+                if not options.quiet:
+                    c.write( "|Checking " + os.path.relpath( root, directory ) )
+
+                for d in dirs:
+                    processed_directory_count += 1
+                    path = join( root, d )
+                    rel_path = os.path.relpath( path, directory )
+
+                    if match_in_ignore_list( path, ignore_list ):
+                        # add option of using send2trash
+                        if not options.quiet:
+                            c.write( "| ignoring " + rel_path )
+                        dirs.remove( d )
+
+                for f in files:
+                    processed_file_count += 1
+                    path = normpath( join( root, f ) )
+
+                    if path not in files_in_depot:
+                        if not options.quiet:
+                            c.write( "| " + f + " is unversioned, removing it." )
+                        try:
+                            os.chmod( path, stat.S_IWRITE )
+                            os.remove( path )
+                            remove_file_count += 1
+                        except OSError as ex:
+                            c.writeflush( "|  " + type( ex ).__name__ )
+                            c.writeflush( "|  " + repr( ex ) )
+                            c.writeflush( "|  ^ERROR^" )
+
+                            error_count += 1
+            if not options.quiet:
+                c.write( "|Done." )
+
+            if not options.quiet:
+                c.write( os.linesep + "Removing empty directories...")
+            # remove empty directories in reverse order
+            for root, dirs, files in os.walk( directory, topdown=False ):
+                ignore_list = get_ignore_list( root, files_to_ignore )
+
+                for d in dirs:
+                    processed_directory_count += 1
+                    path = os.path.join( root, d )
+                    rel_path = os.path.relpath( path, directory )
+
+                    if match_in_ignore_list( path, ignore_list ):
+                        # add option of using send2trash
+                        if not options.quiet:
+                            c.write( "| ignoring " + rel_path )
+                        dirs.remove( d )
+                    try:
+                        os.rmdir(path)
+                        remove_dir_count += 1
+                        if not options.quiet:
+                            c.write( "| " + rel_path + " was removed." )
+                    except OSError:
+                        # Fails on non-empty directory
+                        pass
+            if not options.quiet:
+                c.write( "|Done." )
+
+            if not options.quiet:
+                output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " )
+                output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories")
+                
+                output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " )
+                output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories")
+
+                if warning_count > 0:
+                    output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" )
+                if error_count > 0:
+                    output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )
+
+                end = time.clock()
+                delta = end - start
+                output += "\nFinished in " + str(delta) + "s"
+
+                c.write( output )
 
 if __name__ == "__main__":
     try:
         main( sys.argv )
     except:
-        print( "Unexpected error!" )
+        print( "\nUnexpected error!" )
         traceback.print_exc( file = sys.stdout )
\ No newline at end of file
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
new file mode 100644
index 0000000..f317737
--- /dev/null
+++ b/p4SyncMissingFiles.py
@@ -0,0 +1,136 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+# author              : Brian Ernst
+# python_version      : 2.7.6 and 3.4.0
+# =================================
+
+# TODO: setup batches before pushing to threads and use p4 --parallel
+# http://www.perforce.com/perforce/r14.2/manuals/cmdref/p4_sync.html
+
+from p4Helper import *
+
+import time, traceback
+
+
+#==============================================================
+def main( args ):
+    start = time.clock()
+
+    fail_if_no_p4()
+
+     #http://docs.python.org/library/optparse.html
+    parser = optparse.OptionParser( )
+
+    parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
+    parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 )
+    parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
+    parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
+    parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
+
+    ( options, args ) = parser.parse_args( args )
+
+    directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
+
+    with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
+        with P4Workspace( directory ):
+            if not options.quiet:
+                c.writeflush( "Preparing to sync missing files..." )
+                c.write( " Setting up threads..." )
+
+            # Setup threading
+            WRK = enum( 'SHUTDOWN', 'SYNC' )
+
+            def shutdown( data ):
+                return False
+            def sync( data ):
+                if data is not None and not os.path.exists( data ):
+                    try_call_process( "p4 sync -f " + data )
+                    if not options.quiet:
+                        c.write( " Synced " + data )
+                return True
+
+            commands = {
+                WRK.SHUTDOWN : shutdown,
+                WRK.SYNC : sync
+            }
+
+            threads = [ ]
+            thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
+
+            queue = multiprocessing.JoinableQueue( )
+
+            for i in range( thread_count ):
+                t = Worker( c, queue, commands )
+                threads.append( t )
+                t.start( )
+
+            make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
+
+            command = "p4 fstat ..."
+
+            if not options.quiet:
+                c.writeflush( " Checking files in depot, this may take some time for large depots..." )
+
+            proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory )
+
+            clientFile_tag = "... clientFile "
+            headAction_tag = "... headAction "
+
+            # http://www.perforce.com/perforce/r12.1/manuals/cmdref/fstat.html
+            accepted_actions = [ 'add', 'edit', 'branch', 'move/add', 'move\\add', 'integrate', 'import', 'archive' ] #currently not checked
+            rejected_actions = [ 'delete', 'move/delete', 'move\\delete', 'purge' ]
+
+            client_file = None
+
+            for line in proc.stdout:
+                line = get_str_from_process_stdout( line )
+
+                if client_file and line.startswith( headAction_tag ):
+                    action = normpath( line[ len( headAction_tag ) : ].strip( ) )
+                    if not any(action == a for a in rejected_actions):
+                        if options.verbose:
+                            c.write( " Checking " + os.path.relpath( local_path, directory ) )
+                        queue.put( ( WRK.SYNC, local_path ) )
+
+                if line.startswith( clientFile_tag ):
+                    client_file = None
+                    local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
+                    if make_drive_upper:
+                        drive, path = splitdrive( local_path )
+                        client_file = ''.join( [ drive.upper( ), path ] )
+
+                if len(line.rstrip()) == 0:
+                    client_file = None
+
+            proc.wait( )
+
+            for line in proc.stderr:
+                if "no such file" in line:
+                    continue
+                #raise Exception(line)
+                c.write(line)#log as error
+
+            if not options.quiet:
+                c.writeflush( " Pushed work, now waiting for threads..." )
+
+            for i in range( thread_count ):
+                queue.put( ( WRK.SHUTDOWN, None ) )
+
+            for t in threads:
+                t.join( )
+
+            if not options.quiet:
+                c.write( "Done." )
+
+                end = time.clock()
+                delta = end - start
+                output = "\nFinished in " + str(delta) + "s"
+
+                c.writeflush( output )
+
+if __name__ == "__main__":
+    try:
+        main( sys.argv )
+    except:
+        print( "\nUnexpected error!" )
+        traceback.print_exc( file = sys.stdout )
\ No newline at end of file

From 6610e8e3574f58b84641eb8dc2bb638c21c8d3ad Mon Sep 17 00:00:00 2001
From: unknown 
Date: Wed, 18 Feb 2015 15:14:18 -0700
Subject: [PATCH 19/32] Accidentally committed pyc. Will have to add
 .gitignore.

---
 p4Helper.pyc | Bin 13085 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 p4Helper.pyc

diff --git a/p4Helper.pyc b/p4Helper.pyc
deleted file mode 100644
index b4884754adb315b7046b77cc97aafc988f1eac1b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 13085
zcmcgyO>i8?b?%v6{4B5_Kma5NiWbKtC4m$P%4JA$Xj!HNfIqghga)8jMv)p1X9mOo
zyEBXFSpskXmnE_sIkBC;N}S5CQmLeJNK%zUPR>15IpmN-ZmyhCm2+~B^L?-PPb8d#
zgcXR}dehTCuV26Sz1Od&@Sh_?b@gv6folB~@&Bv%X=N)F;-95zO09NiORZY%dRwj9
zZMvXV3vIfnR*P-APp$T~>3+4^-=+uD>Oh+wRI7t+x};W1ZF)$p4z=lFwK}YJhg4Kn
zt0StK3@FR85=*@w^{b5|Dzw!@ur{i~f_emg$5dDp;;0Jygcw)RxC${prX5rd!S93$
zG25hy`qaiT^$@*|t8hpfPpNQNh!ZL-3o)(25g|^h@Q4u4sc=+?Qz{%2;&YT%cvL+C
zC#O~9sBoNIMW0ujnCOfOC#2a~X*Ma%&Ix@?=oeIYTpGNn!YLtMQsD_9W>g3U(D5W6
z!Rj|x&#Uk`6EoR_ooa9BWRD3Ge)g;FyGHsNy?
z<)n4-8wfXE<`sLeHR3Qs|dKUAuB;DN8q^AlG*qbFTEdojiy#
z9j8eYqV7giYed;>V>ds7_MaY?JZdtatbh#g$H#g6^dt%}4c0epm5r%AEPc;zuOKEj
zh8gw}{vY_Ye2`t^q
zC-=?1)v!fulJ7Pmv#CRP($Q8EFJcIV%@e;Kg+fIb{4o2-sW`@SoP-a`I%kRCmv7u&
zS-4fLud>wItMp4NVUp@P#yC*ONyPHieJso%M6j>ic
zftA&T6KbnFjp7hIokaI9;@4I@N{T-XS=oM!be!n?ugc-~VY(4SS-wjarW;1Ui)z2G)pVms>)jJeo^oL0`~7MU{{UM3
zDtkfg71Tx_A5l4t*Vfhx3jeC8Z$YsEGJ=O}f#%u;lv8a$Icnsz6k1%A`FZ40%3=V~
zP97!U4A&wu60=*2YmxTy)H8O~!j4Sm!)RveyjrYtf%DjviD{bLdv;rkF(jU!5#(w4
zfTh;#GSgR3r3wDZ9=H4LY3rn2woceVTX(6EvJZ49JLf2^g}bK&A&vP%z`##9fdRtl
z^45T=LNsmFW_OobCi1KMd9W^vUdft~R9*9H+Tfx{$$gP+*&#c`Iw#Rz=JMjCcd=hb
zy&goNtPyM68nimlK}pLpUjWfDQ>vRdIo2RngIf^dp=Gxn0z2wFOq;pvI>;!?X1MHJ
ztfhIck)=VTwJbaEYc=;YU$Vv0&85XIDEis`!vH_kgu{ES`IhnAHg#?$f1Oowsk<*7
zv`|?k;mNWs1cuAiAq_Q^|Lxx4Q{C=pg$+XgnF^%=8Ruh{Dw!;|J%GeHD
zxL-gaE+$xyf=v&?YBY0^gL{#$#!<|J_b5cXLSZdtS(*t5x^o=haNwK^_GY_YuYA6N
zaW%I{Sq1AfbYaXgCY@&;t>7k!R-6L?A!LYK0YVbBp|MeX0LL<@Wf2IaTyaq-qXVZv
zHFf0O+sz|C%lutW=NTfmb`)mtqJUO)zhT6;o^^Gi7AMgm?0{fId*_^MSz2%F1}dnr
zAKe)O
zu+7LoIrV~O=6)5#A@+x%NW;&dA91nm6^?Txb)
zcE;?ZwRKHZtI1JdLP-GOVf#>71;?h>#t^sBdLLS`JVW)Wh|6ru2^?Lz{@Aw7(D%u%
zxo?poJi4-BbUDxt+D4|=5h_#ewbTz~1i@Rsfjtf-oLjtgO<+SdE2yP#D*`@q9FG7Z
zLuHX})^hD$V+ENrV04I0u3Im9{#&<%RFKM=qxlGtKSFmN)Twy
zo%#YfglD3>5P`fxW0E?$8LT@DYiD>WGYS*suEPhKD-V9|dwk-SHfnwjovXW4ck%rP8ouEA
zRW^B>1-*cPv`dw6d1Nqtlj`3D0%S6;ABHmJwjbx>6(lrY+zAk*BjGVay)r<+eiDjH
zWKj1vBTr5=o*9gg(}H2a34wXZkaVTs<^Fx~BefJ!g~tG0fXAGSFu*g0=tt0#Xkupa
zbd-M|H6}2G7r1~J-S~rk3m(Bf1$Y^^p0|(MLpHnwp2qBevvY}rH%d%%2$&4ij_}{>
z&D#+zjKz2c#C@=`o!g><^y6O>*8+cG)Dw1v6PCDq<0fbbVauDtqR>*kGOH@hHO!f8KFNs5MZJ^y}$?0!tu9{|>y5R?rY#^rx_909bX<~
z-dRUFybF7PM`&j37dcV#AkNZ+sYaaD93R!t%4Wj}(hRAgytaGMLBN8xK#<+E$ri~=
zd1sE4b0leU33h5cQwZM^d!h@5Pc!MC{n2y5ZxP~MLh#cdx_ZS<~y#QS`17E
zsjC0TX2aZXOl<~QQ9YTh{#lM0R
zpaBqrrYkyH#}5Jk!HN(BPKl=HI!wSXuds
z=9~F8I0)Z8@FqUPAc!P~0CL2)HXd}Wt`!w`-%uvAGemWFi3J%K#x|o}Ey0b;n8>}#
z0&1$nQlSbwt!zh`yTCdExqF?(*I8UaAs0GnE$m3alRP=cccHN6fMQM*5nn4uL93RNguGYI|1tVwIKSdysbOkuPzhFE3
zG;nNX`=E6JqYk+O`1G6i{9u!~pJws>IPq((@Ys0}N85;&4_3~b7-NNg#M$=YN8vQm
zd$p+UFA#BkVZxJBc3so2nOv#T<&Gt=3i^R1KG0EU#^6%~_h3(U4(B$)$$B%7c1$XikRbV}u3(;8CY10|tfh4~IA+P6qzjgniv=rT
ztvmj+3I2N+XpECkVEvY?V`%|*qye7gkJ2O{>d#i(G
zZ@81~7xV8dzU4moLBWT7FNae^lt5>qaBS?TpjrfSHwf8)76|X!5h5QEy2-(aX#Wmr
z8B*7RibrBOQ^)=k}5{cexMy>lWd=`K#
z^8y<%4Dg%0^{X}Yx`!)9fsCFi4sUKyIGA3?B@~7SmTo7;tOYTCjCbzuq3D|RGr`6g
zDj>ry1vgxh|3
zH7F=HUG@PpA)P;n(*0c&lHTE!(GS`W?=D|=f5_LAl*u-6V{qzj#9?>sg1DiN+!lA1
zukgQTJJq}2MbC?E8&n4@`YsxDwGGKVvqW=iEg5=>L?i({M)m|tN1N`_AAgNJimlcp
z@6s!l66Wcy+xd|3tfQ4_UOWsEAA-5<4fe~9uq|mvLAAt1XNk#T+gomzb^0B
zKkMZMHQY+R-m^TAl6B($#gYj^X7^yF7N~YW!xnI?W0007c20;yctio%K-%&eX6d|<1bK&k8
z`Sb-B9=OApU}|LH?l6-l$RAw4u~k*otp(g6U;=cJ+=9re)cb&`&Ar`H@)N5&<*=F3
zB0IJ^eFw=EOd}VBW;~{T{)P85<`HRGY}WT?;w-&d|&s@nN*}jx?@TjOW-|5*gx?Rnv9JK5*LU6S#6=
z$8j0Nbm|=>`ALt+CuRL*+pK?@nQybSez|j5vUg7Qu=gpk{}ZlWWO=x`K_@7r91#u_
z1$a*udZwL{X~CO_{eWo?7@1=j@pkrdtDvf42voTW!l?FV8N949moOdw3g6k|Jj679
z3$4V)0rqw6{wX{CHtXnua$j#E)JP(6G;+j)r;Qksh__ABZIkJc_=7m&e!_y0h
36Y!S=sFyRMj?T0)!<7pCo4Mv_Yu
z0d=zDeyVubS>nPnSIoW**u+IiZPPBXKp;0(3$)U2x7#7J1dUqiny9Jhm?$T!m^-PL
zL7GMV87Z#J{c{xEKu&ZusD~2J9b(ngt&h3tG6?GXF=+3IRkVzx!EsMo;1{Bae~lqO
zXST1Mi1^Hm`S;nlUdMt+aY
zMTe%?hbWc#+qm+;*}>qLQ1rpjs0MDWU2d<-NY7%m@M3zl!(5H;t>F$poZ|lO0hU
Date: Wed, 18 Feb 2015 15:28:58 -0700
Subject: [PATCH 20/32] Fixed output bugs in p4SyncMissingFiles.py.

---
 p4SyncMissingFiles.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index f317737..3863dd2 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -44,9 +44,9 @@ def main( args ):
                 return False
             def sync( data ):
                 if data is not None and not os.path.exists( data ):
-                    try_call_process( "p4 sync -f " + data )
+                    subprocess.check_output( "p4 sync -f \"" + data + "\"", shell=False, cwd=None )
                     if not options.quiet:
-                        c.write( " Synced " + data )
+                        c.write( "  Synced " + os.path.relpath( data, directory ) )
                 return True
 
             commands = {
@@ -89,7 +89,8 @@ def main( args ):
                     action = normpath( line[ len( headAction_tag ) : ].strip( ) )
                     if not any(action == a for a in rejected_actions):
                         if options.verbose:
-                            c.write( " Checking " + os.path.relpath( local_path, directory ) )
+                            c.write( "  Checking " + os.path.relpath( local_path, directory ) )
+                        # TODO: directories should be batched and synced in parallel
                         queue.put( ( WRK.SYNC, local_path ) )
 
                 if line.startswith( clientFile_tag ):

From c32c0bfbd15f80545a21a67b06bf3667f6def506 Mon Sep 17 00:00:00 2001
From: unknown 
Date: Tue, 12 May 2015 14:47:18 -0600
Subject: [PATCH 21/32] Added bucketing based on file type (text/binary) and
 batching to reduce server calls.

---
 p4Helper.py           |  25 ++---
 p4SyncMissingFiles.py | 229 ++++++++++++++++++++++++++----------------
 2 files changed, 156 insertions(+), 98 deletions(-)

diff --git a/p4Helper.py b/p4Helper.py
index d50bd63..a22d862 100644
--- a/p4Helper.py
+++ b/p4Helper.py
@@ -4,7 +4,7 @@
 # python_version      : 2.7.6 and 3.4.0
 # =================================
 
-import datetime, inspect, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading
+import datetime, inspect, itertools, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading
 
 # trying ntpath, need to test on linux
 import ntpath
@@ -297,19 +297,19 @@ class Console( threading.Thread ):
         self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
         self.shutting_down = False
 
-    def write( self, data, pid = None ):
-        self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) )
+    def write( self, data ):
+        self.queue.put( ( Console.MSG.WRITE, threading.current_thread().ident, data ) )
 
-    def writeflush( self, data, pid = None ):
-        pid = pid if pid is not None else os.getpid()
+    def writeflush( self, data ):
+        pid = threading.current_thread().ident
         self.queue.put( ( Console.MSG.WRITE, pid, data ) )
         self.queue.put( ( Console.MSG.FLUSH, pid ) )
 
-    def flush( self, pid = None ):
-        self.queue.put( ( Console.MSG.FLUSH, pid if pid is not None else os.getpid() ) )
+    def flush( self ):
+        self.queue.put( ( Console.MSG.FLUSH, threading.current_thread().ident ) )
 
-    def clear( self, pid = None ):
-        self.queue.put( ( Console.MSG.CLEAR, pid if pid is not None else os.getpid() ) )
+    def clear( self ):
+        self.queue.put( ( Console.MSG.CLEAR, threading.current_thread().ident ) )
 
     def __enter__( self ):
         self.start( )
@@ -325,16 +325,12 @@ class Console( threading.Thread ):
             event = data[0]
 
             if event == Console.MSG.SHUTDOWN:
-                # flush remaining buffers before shutting down
                 for ( pid, buffer ) in self.buffers.items( ):
                     for line in buffer:
                         print( line )
                 self.buffers.clear( )
                 self.buffer_write_times.clear( )
                 self.queue.task_done( )
-
-                #print(self.queue.qsize())
-                #print(self.queue.empty())
                 break
 
             elif event == Console.MSG.WRITE:
@@ -354,7 +350,8 @@ class Console( threading.Thread ):
             elif event == Console.MSG.FLUSH:
                 pid = data[ 1 ]
                 if pid in self.buffers:
-                    for line in self.buffers[ pid ]:
+                    buffer = self.buffers[ pid ]
+                    for line in buffer:
                         print( line )
                     self.buffers.pop( pid, None )
                     self.buffer_write_times[ pid ] = datetime.datetime.now( )
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index 3863dd2..f39a636 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -13,125 +13,186 @@ import time, traceback
 
 
 #==============================================================
-def main( args ):
-    start = time.clock()
+class P4SyncMissing:
+    def run( self, args ):
+        start = time.clock()
 
-    fail_if_no_p4()
+        fail_if_no_p4()
 
-     #http://docs.python.org/library/optparse.html
-    parser = optparse.OptionParser( )
+         #http://docs.python.org/library/optparse.html
+        parser = optparse.OptionParser( )
 
-    parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
-    parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 )
-    parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
-    parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
-    parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
+        parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
+        parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=12 )
+        parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
+        parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
+        parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
 
-    ( options, args ) = parser.parse_args( args )
+        ( options, args ) = parser.parse_args( args )
 
-    directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
+        directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
 
-    with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
-        with P4Workspace( directory ):
-            if not options.quiet:
-                c.writeflush( "Preparing to sync missing files..." )
-                c.write( " Setting up threads..." )
+        with Console( auto_flush_time=1000 ) as c:
+            with P4Workspace( directory ):
+                if not options.quiet:
+                    c.writeflush( "Preparing to sync missing files..." )
+                    c.writeflush( " Setting up threads..." )
 
-            # Setup threading
-            WRK = enum( 'SHUTDOWN', 'SYNC' )
+                # Setup threading
+                WRK = enum( 'SHUTDOWN', 'SYNC' )
+
+                def shutdown( data ):
+                    return False
+                def sync( files ):
+                    files_flat = ' '.join(files)
+                    #subprocess.check_output( "p4 sync -f " + files_flat + "", shell=False, cwd=None )
+                    ret = -1
+                    count = 0
+                    while ret != 0 and count < 2:
+                        ret = try_call_process( "p4 sync -f " + files_flat )
+                        count += 1
+                        if ret != 0 and not options.quiet:
+                            c.write("Failed, trying again to sync " + files_flat)
+                    
 
-            def shutdown( data ):
-                return False
-            def sync( data ):
-                if data is not None and not os.path.exists( data ):
-                    subprocess.check_output( "p4 sync -f \"" + data + "\"", shell=False, cwd=None )
                     if not options.quiet:
-                        c.write( "  Synced " + os.path.relpath( data, directory ) )
-                return True
+                        files_len = len(files)
+                        if files_len > 1:
+                            c.write( "  Synced batch of " + str(len(files)) )
+                        for f in files:
+                            c.write( "   Synced " + os.path.relpath( f, directory ) )
+                    return True
 
-            commands = {
-                WRK.SHUTDOWN : shutdown,
-                WRK.SYNC : sync
-            }
+                commands = {
+                    WRK.SHUTDOWN : shutdown,
+                    WRK.SYNC : sync
+                }
 
-            threads = [ ]
-            thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
+                threads = [ ]
+                thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
 
-            queue = multiprocessing.JoinableQueue( )
+                count = 0
+                self.queue = multiprocessing.JoinableQueue( )
 
-            for i in range( thread_count ):
-                t = Worker( c, queue, commands )
-                threads.append( t )
-                t.start( )
+                for i in range( thread_count ):
+                    t = Worker( c, self.queue, commands )
+                    threads.append( t )
+                    t.start( )
 
-            make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
+                make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
 
-            command = "p4 fstat ..."
+                command = "p4 fstat ..."
 
-            if not options.quiet:
-                c.writeflush( " Checking files in depot, this may take some time for large depots..." )
+                if not options.quiet:
+                    c.writeflush( " Checking files in depot, this may take some time for large depots..." )
 
-            proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory )
+                proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory )
 
-            clientFile_tag = "... clientFile "
-            headAction_tag = "... headAction "
+                clientFile_tag = "... clientFile "
+                headAction_tag = "... headAction "
+                headType_tag   = "... headType "
 
-            # http://www.perforce.com/perforce/r12.1/manuals/cmdref/fstat.html
-            accepted_actions = [ 'add', 'edit', 'branch', 'move/add', 'move\\add', 'integrate', 'import', 'archive' ] #currently not checked
-            rejected_actions = [ 'delete', 'move/delete', 'move\\delete', 'purge' ]
+                # http://www.perforce.com/perforce/r12.1/manuals/cmdref/fstat.html
+                accepted_actions = [ 'add', 'edit', 'branch', 'move/add', 'move\\add', 'integrate', 'import', 'archive' ] #currently not checked
+                rejected_actions = [ 'delete', 'move/delete', 'move\\delete', 'purge' ]
+                file_type_binary = 'binary+l'
+                file_type_text   = 'text'
 
-            client_file = None
+                client_file = None
+                file_action = None
+                file_type   = None
+                file_type_last = None
 
-            for line in proc.stdout:
-                line = get_str_from_process_stdout( line )
+                class Bucket:
+                    def __init__(self, limit):
+                        self.queue = []
+                        self.queue_size = 0
+                        self.queue_limit = limit
+                    def append(self,obj):
+                        self.queue.append(obj)
+                        self.queue_size += 1
+                    def is_full(self):
+                        return self.queue_size >= self.queue_limit
 
-                if client_file and line.startswith( headAction_tag ):
-                    action = normpath( line[ len( headAction_tag ) : ].strip( ) )
-                    if not any(action == a for a in rejected_actions):
-                        if options.verbose:
-                            c.write( "  Checking " + os.path.relpath( local_path, directory ) )
-                        # TODO: directories should be batched and synced in parallel
-                        queue.put( ( WRK.SYNC, local_path ) )
+                self.buckets             = {}
+                self.buckets[file_type_text]   = Bucket(10)
+                self.buckets[file_type_binary] = Bucket(2)
 
-                if line.startswith( clientFile_tag ):
-                    client_file = None
-                    local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
-                    if make_drive_upper:
-                        drive, path = splitdrive( local_path )
-                        client_file = ''.join( [ drive.upper( ), path ] )
+                def push_queued(bucket):
+                    if bucket.queue_size == 0:
+                        return
+                    if options.verbose:
+                        for f in bucket.queue:
+                            c.write( "  Checking " + os.path.relpath( f, directory ) )
+                    self.queue.put( ( WRK.SYNC, bucket.queue ) )
+                    bucket.queue = []
+                    bucket.queue_size = 0
 
-                if len(line.rstrip()) == 0:
-                    client_file = None
+                for line in proc.stdout:
+                    line = get_str_from_process_stdout( line )
 
-            proc.wait( )
+                    #push work when finding out type
+                    if client_file and file_action is not None and line.startswith( headType_tag ):
 
-            for line in proc.stderr:
-                if "no such file" in line:
-                    continue
-                #raise Exception(line)
-                c.write(line)#log as error
+                        file_type = normpath( line[ len( headType_tag ) : ].strip( ) )
+                        if file_type == file_type_text:
+                            self.buckets[file_type_text].append(client_file)
+                        else:
+                            self.buckets[file_type_binary].append(client_file)
+                        count += 1
 
-            if not options.quiet:
-                c.writeflush( " Pushed work, now waiting for threads..." )
+                        #check sizes and push
+                        for b in self.buckets.values():
+                            if b.is_full():
+                                push_queued(b)
+                    
+                    elif client_file and line.startswith( headAction_tag ):
+                        file_action = normpath( line[ len( headAction_tag ) : ].strip( ) )
+                        if any(file_action == a for a in rejected_actions):
+                            file_action = None
+                        else:
+                            if os.path.exists( client_file ):
+                                file_action = None
 
-            for i in range( thread_count ):
-                queue.put( ( WRK.SHUTDOWN, None ) )
+                    elif line.startswith( clientFile_tag ):
+                        client_file = normpath( line[ len( clientFile_tag ) : ].strip( ) )
+                        if make_drive_upper:
+                            drive, path = splitdrive( client_file )
+                            client_file = ''.join( [ drive.upper( ), path ] )
 
-            for t in threads:
-                t.join( )
+                    elif len(line.rstrip()) == 0:
+                        client_file = None
 
-            if not options.quiet:
-                c.write( "Done." )
+                for b in self.buckets.values():
+                    push_queued(b)
+                proc.wait( )
 
-                end = time.clock()
-                delta = end - start
-                output = "\nFinished in " + str(delta) + "s"
+                for line in proc.stderr:
+                    if "no such file" in line:
+                        continue
+                    #raise Exception(line)
+                    c.write(line)#log as error
 
-                c.writeflush( output )
+                if not options.quiet:
+                    c.writeflush( " Checking " + str(count) + " file(s), now waiting for threads..." )
+
+                for i in range( thread_count ):
+                    self.queue.put( ( WRK.SHUTDOWN, None ) )
+
+                for t in threads:
+                    t.join( )
+
+        if not options.quiet:
+            print( "Done." )
+
+            end = time.clock()
+            delta = end - start
+            output = "\nFinished in " + str(delta) + "s"
+            print( output )
 
 if __name__ == "__main__":
     try:
-        main( sys.argv )
+        P4SyncMissing().run(sys.argv)
     except:
         print( "\nUnexpected error!" )
         traceback.print_exc( file = sys.stdout )
\ No newline at end of file

From ea14f96d7645edaf4d72d859987e6a069ae60aa5 Mon Sep 17 00:00:00 2001
From: unknown 
Date: Wed, 13 May 2015 10:45:04 -0600
Subject: [PATCH 22/32] Fixed bug in p4SyncMissingFiles.py. Also fixed bug in
 p4Helper when running p4RemoveUnversioned.py.

---
 p4Helper.py           | 35 +++++++++++++++++++++--------------
 p4SyncMissingFiles.py | 28 +++++++++++++++-------------
 2 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/p4Helper.py b/p4Helper.py
index a22d862..5c20a3b 100644
--- a/p4Helper.py
+++ b/p4Helper.py
@@ -85,7 +85,7 @@ def call_process( args ):
 
 def try_call_process( args, path=None ):
     try:
-        subprocess.check_output( args.split( ), shell=False, cwd=path )
+        subprocess.check_output( args.split( ), shell=False, cwd=path, stderr=subprocess.STDOUT )
         return 0
     except subprocess.CalledProcessError:
         return 1
@@ -204,7 +204,7 @@ class P4Workspace:
         #print("\nChecking p4 info...")
         result = get_p4_py_results('info')
         if len(result) == 0 or b'userName' not in result[0].keys():
-            print("Can't find perforce info, is it even setup?")
+            print("Can't find perforce info, is it even setup? Possibly can't connect to server.")
             sys.exit(1)
         username = get_str_from_process_stdout(result[0][b'userName'])
         client_host = get_str_from_process_stdout(result[0][b'clientHost'])
@@ -297,19 +297,22 @@ class Console( threading.Thread ):
         self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
         self.shutting_down = False
 
-    def write( self, data ):
-        self.queue.put( ( Console.MSG.WRITE, threading.current_thread().ident, data ) )
+    def write( self, data, pid = None ):
+        pid = pid if pid is not None else threading.current_thread().ident
+        self.queue.put( ( Console.MSG.WRITE, pid, data ) )
 
-    def writeflush( self, data ):
-        pid = threading.current_thread().ident
+    def writeflush( self, data, pid = None ):
+        pid = pid if pid is not None else threading.current_thread().ident
         self.queue.put( ( Console.MSG.WRITE, pid, data ) )
         self.queue.put( ( Console.MSG.FLUSH, pid ) )
 
-    def flush( self ):
-        self.queue.put( ( Console.MSG.FLUSH, threading.current_thread().ident ) )
+    def flush( self, pid = None ):
+        pid = pid if pid is not None else threading.current_thread().ident
+        self.queue.put( ( Console.MSG.FLUSH, pid ) )
 
-    def clear( self ):
-        self.queue.put( ( Console.MSG.CLEAR, threading.current_thread().ident ) )
+    def clear( self, pid = None ):
+        pid = pid if pid is not None else threading.current_thread().ident
+        self.queue.put( ( Console.MSG.CLEAR, pid ) )
 
     def __enter__( self ):
         self.start( )
@@ -342,10 +345,14 @@ class Console( threading.Thread ):
                     self.buffer_write_times[ pid ] = datetime.datetime.now( )
                 self.buffers[ pid ].append( s )
 
-                if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num:
-                    self.flush( pid )
-                elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time:
-                    self.flush( pid )
+                try:
+                    if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num:
+                        self.flush( pid )
+                    elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time:
+                        self.flush( pid )
+                except TypeError:
+                    print('"' + pid + '"')
+                    raise
                 # TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time
             elif event == Console.MSG.FLUSH:
                 pid = data[ 1 ]
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index f39a636..28da004 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -44,23 +44,25 @@ class P4SyncMissing:
                 def shutdown( data ):
                     return False
                 def sync( files ):
-                    files_flat = ' '.join(files)
-                    #subprocess.check_output( "p4 sync -f " + files_flat + "", shell=False, cwd=None )
+                    files_flat = ' '.join('"' + f + '"' for f in files)
                     ret = -1
                     count = 0
                     while ret != 0 and count < 2:
                         ret = try_call_process( "p4 sync -f " + files_flat )
                         count += 1
-                        if ret != 0 and not options.quiet:
-                            c.write("Failed, trying again to sync " + files_flat)
-                    
-
-                    if not options.quiet:
-                        files_len = len(files)
-                        if files_len > 1:
-                            c.write( "  Synced batch of " + str(len(files)) )
-                        for f in files:
-                            c.write( "   Synced " + os.path.relpath( f, directory ) )
+                        #if ret != 0 and not options.quiet:
+                        #    c.write("Failed, trying again to sync " + files_flat)
+                    if ret != 0:
+                        pass
+                        #if not options.quiet:
+                        #    c.write("Failed to sync " + files_flat)
+                    else:
+                        if not options.quiet:
+                            files_len = len(files)
+                            if files_len > 1:
+                                c.write( "  Synced batch of " + str(len(files)) )
+                            for f in files:
+                                c.write( "   Synced " + os.path.relpath( f, directory ) )
                     return True
 
                 commands = {
@@ -116,7 +118,7 @@ class P4SyncMissing:
 
                 self.buckets             = {}
                 self.buckets[file_type_text]   = Bucket(10)
-                self.buckets[file_type_binary] = Bucket(2)
+                self.buckets[file_type_binary] = Bucket(1)
 
                 def push_queued(bucket):
                     if bucket.queue_size == 0:

From 92d217371cbb0669fdf82ae4961a81989969ccef Mon Sep 17 00:00:00 2001
From: unknown 
Date: Wed, 13 May 2015 12:06:54 -0600
Subject: [PATCH 23/32] Fixed scripts up, improved logging so console has a
 waking thread now. Also fixed bug if console timer is too long it'll be
 killed off appropriately.

---
 p4Helper.py            | 21 ++++++++++++++++++---
 p4RemoveUnversioned.py |  2 +-
 p4SyncMissingFiles.py  | 41 +++++++++++++++++++++++++++++++----------
 3 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/p4Helper.py b/p4Helper.py
index 5c20a3b..a82f845 100644
--- a/p4Helper.py
+++ b/p4Helper.py
@@ -81,11 +81,11 @@ def match_in_ignore_list( path, ignore_list ):
 
 #==============================================================
 def call_process( args ):
-    return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
+    return subprocess.call( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
 
 def try_call_process( args, path=None ):
     try:
-        subprocess.check_output( args.split( ), shell=False, cwd=path, stderr=subprocess.STDOUT )
+        subprocess.check_output( args, shell=False, cwd=path )#, stderr=subprocess.STDOUT )
         return 0
     except subprocess.CalledProcessError:
         return 1
@@ -113,7 +113,7 @@ def parse_info_from_command( args, value, path = None ):
 
 def get_p4_py_results( args, path = None ):
     results = []
-    proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
+    proc = subprocess.Popen( 'p4 -G ' + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
     try:
         while True:
             output = marshal.load( proc.stdout )
@@ -286,6 +286,14 @@ class PDict( dict ):
 class Console( threading.Thread ):
     MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' )
 
+    @staticmethod
+    def wake(thread):
+        thread.flush()
+        if not thread.shutting_down:
+            thread.wake_thread = threading.Timer(thread.auto_flush_time / 1000.0, Console.wake, [thread])
+            thread.wake_thread.daemon = True
+            thread.wake_thread.start()
+
     # auto_flush_time is time in milliseconds since last flush to trigger a flush when writing
     def __init__( self, auto_flush_num = None, auto_flush_time = None ):
         threading.Thread.__init__( self )
@@ -296,6 +304,9 @@ class Console( threading.Thread ):
         self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1
         self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
         self.shutting_down = False
+        self.wake_thread = None
+        if self.auto_flush_time > 0:
+            Console.wake(self)
 
     def write( self, data, pid = None ):
         pid = pid if pid is not None else threading.current_thread().ident
@@ -319,6 +330,10 @@ class Console( threading.Thread ):
         return self
 
     def __exit__( self, type, value, tb ):
+        self.shutting_down = True
+        if self.wake_thread:
+            self.wake_thread.cancel()
+            self.wake_thread.join()
         self.queue.put( ( Console.MSG.SHUTDOWN, ) )
         self.queue.join( )
 
diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py
index a816cc4..eaaa960 100644
--- a/p4RemoveUnversioned.py
+++ b/p4RemoveUnversioned.py
@@ -35,7 +35,7 @@ def main( args ):
 
     directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
 
-    with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
+    with Console( auto_flush_time=1 ) as c:
         with P4Workspace( directory ):
             # Files are added from .p4ignore
             # Key is the file root, the value is the table of file regexes for that directory.
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index 28da004..c5b6d33 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -32,10 +32,10 @@ class P4SyncMissing:
 
         directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
 
-        with Console( auto_flush_time=1000 ) as c:
+        with Console( auto_flush_time=1 ) as c:
             with P4Workspace( directory ):
                 if not options.quiet:
-                    c.writeflush( "Preparing to sync missing files..." )
+                    c.writeflush( "Retreiving missing files..." )
                     c.writeflush( " Setting up threads..." )
 
                 # Setup threading
@@ -44,18 +44,29 @@ class P4SyncMissing:
                 def shutdown( data ):
                     return False
                 def sync( files ):
+                    files_len = len(files)
                     files_flat = ' '.join('"' + f + '"' for f in files)
+
+                    if options.verbose:
+                        files_len = len(files)
+                        if files_len > 1:
+                            c.write( "  Syncing batch of " + str(len(files)) + " ...")
+                            for f in files:
+                                c.write( "   " + os.path.relpath( f, directory ) )
+                        else:
+                            for f in files:
+                                c.write( "   Syncing " + os.path.relpath( f, directory ) + " ..." )
+
                     ret = -1
                     count = 0
                     while ret != 0 and count < 2:
                         ret = try_call_process( "p4 sync -f " + files_flat )
                         count += 1
-                        #if ret != 0 and not options.quiet:
-                        #    c.write("Failed, trying again to sync " + files_flat)
+                        if ret != 0 and not options.quiet:
+                            c.write("Failed, trying again to sync " + files_flat)
                     if ret != 0:
-                        pass
-                        #if not options.quiet:
-                        #    c.write("Failed to sync " + files_flat)
+                        if not options.quiet:
+                            c.write("Failed to sync " + files_flat)
                     else:
                         if not options.quiet:
                             files_len = len(files)
@@ -74,13 +85,17 @@ class P4SyncMissing:
                 thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
 
                 count = 0
+                total = 0
                 self.queue = multiprocessing.JoinableQueue( )
 
                 for i in range( thread_count ):
                     t = Worker( c, self.queue, commands )
+                    t.daemon = True
                     threads.append( t )
                     t.start( )
 
+                c.writeflush( "  Done." )
+
                 make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
 
                 command = "p4 fstat ..."
@@ -105,6 +120,7 @@ class P4SyncMissing:
                 file_type   = None
                 file_type_last = None
 
+                # todo: use fewer threads, increase bucket size and use p4 threading
                 class Bucket:
                     def __init__(self, limit):
                         self.queue = []
@@ -118,7 +134,7 @@ class P4SyncMissing:
 
                 self.buckets             = {}
                 self.buckets[file_type_text]   = Bucket(10)
-                self.buckets[file_type_binary] = Bucket(1)
+                self.buckets[file_type_binary] = Bucket(2)
 
                 def push_queued(bucket):
                     if bucket.queue_size == 0:
@@ -153,11 +169,12 @@ class P4SyncMissing:
                         if any(file_action == a for a in rejected_actions):
                             file_action = None
                         else:
+                            total += 1
                             if os.path.exists( client_file ):
                                 file_action = None
 
                     elif line.startswith( clientFile_tag ):
-                        client_file = normpath( line[ len( clientFile_tag ) : ].strip( ) )
+                        client_file = line[ len( clientFile_tag ) : ].strip( )
                         if make_drive_upper:
                             drive, path = splitdrive( client_file )
                             client_file = ''.join( [ drive.upper( ), path ] )
@@ -176,7 +193,8 @@ class P4SyncMissing:
                     c.write(line)#log as error
 
                 if not options.quiet:
-                    c.writeflush( " Checking " + str(count) + " file(s), now waiting for threads..." )
+                    c.writeflush( "  Done. Checked " + str(total) + " file(s)." )
+                    c.writeflush( " Queued " + str(count) + " file(s), now waiting for threads..." )
 
                 for i in range( thread_count ):
                     self.queue.put( ( WRK.SHUTDOWN, None ) )
@@ -184,6 +202,9 @@ class P4SyncMissing:
                 for t in threads:
                     t.join( )
 
+                if not options.quiet:
+                    print( "  Done." )
+
         if not options.quiet:
             print( "Done." )
 

From 26d1127e64b798ac51189aab88f923fc4346f3ea Mon Sep 17 00:00:00 2001
From: unknown 
Date: Wed, 13 May 2015 12:10:58 -0600
Subject: [PATCH 24/32] Neatened up console output a little bit.

---
 p4Helper.py           | 6 +++---
 p4SyncMissingFiles.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/p4Helper.py b/p4Helper.py
index a82f845..937b13b 100644
--- a/p4Helper.py
+++ b/p4Helper.py
@@ -102,7 +102,7 @@ def parse_info_from_command( args, value, path = None ):
 
     :rtype : string
     """
-    proc = subprocess.Popen( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
+    proc = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
     for line in proc.stdout:
         line = get_str_from_process_stdout( line )
 
@@ -141,7 +141,7 @@ def get_client_set( path ):
 
     command = "p4 fstat ..."
 
-    proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
+    proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
     for line in proc.stdout:
         line = get_str_from_process_stdout( line )
 
@@ -172,7 +172,7 @@ def get_client_root( ):
     """
     command = "p4 info"
 
-    proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
+    proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
     for line in proc.stdout:
         line = get_str_from_process_stdout( line )
 
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index c5b6d33..0991738 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -206,7 +206,7 @@ class P4SyncMissing:
                     print( "  Done." )
 
         if not options.quiet:
-            print( "Done." )
+            print( " Done." )
 
             end = time.clock()
             delta = end - start

From a5f82d5e00346f97d6ab991c16a40cbec6d60512 Mon Sep 17 00:00:00 2001
From: unknown 
Date: Wed, 13 May 2015 12:12:36 -0600
Subject: [PATCH 25/32] Neatened up output.

---
 p4SyncMissingFiles.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index 0991738..15d418a 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -115,9 +115,9 @@ class P4SyncMissing:
                 file_type_binary = 'binary+l'
                 file_type_text   = 'text'
 
-                client_file = None
-                file_action = None
-                file_type   = None
+                client_file    = None
+                file_action    = None
+                file_type      = None
                 file_type_last = None
 
                 # todo: use fewer threads, increase bucket size and use p4 threading
@@ -206,11 +206,9 @@ class P4SyncMissing:
                     print( "  Done." )
 
         if not options.quiet:
-            print( " Done." )
-
             end = time.clock()
             delta = end - start
-            output = "\nFinished in " + str(delta) + "s"
+            output = " Done. Finished in " + str(delta) + "s"
             print( output )
 
 if __name__ == "__main__":

From d5dc8155f5b527f48742e26f0df6a8c928fe5812 Mon Sep 17 00:00:00 2001
From: unknown 
Date: Mon, 8 Jun 2015 14:50:07 -0600
Subject: [PATCH 26/32] Fixed up path limitation issue with p4.

---
 .gitignore            |  1 +
 p4Helper.py           | 15 +++++++++++
 p4Sync.py             | 62 +++++++++++++++++++++++++++++++++++++++++++
 p4SyncMissingFiles.py |  5 ++--
 4 files changed, 80 insertions(+), 3 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 p4Sync.py

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7e99e36
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.pyc
\ No newline at end of file
diff --git a/p4Helper.py b/p4Helper.py
index 937b13b..b11d768 100644
--- a/p4Helper.py
+++ b/p4Helper.py
@@ -57,6 +57,21 @@ def join( patha, pathb ):
 def splitdrive( path ):
     return ntpath.splitdrive( path )
 
+def p4FriendlyPath(path):
+    """
+    Returns path with sanitized unsupported characters due to filename limitations.
+    """
+    # http://www.perforce.com/perforce/doc.current/manuals/cmdref/filespecs.html#1041962
+    replace_items = {
+        '@' : '%40',
+        '#' : '%23',
+        '*' : '%2A',
+        '%' : '%25'
+    }
+    def replace(c):
+        return replace_items[c] if c in replace_items else c
+    return ''.join(map(replace, path))
+
 #==============================================================
 def get_ignore_list( path, files_to_ignore ):
     # have to split path and test top directory
diff --git a/p4Sync.py b/p4Sync.py
new file mode 100644
index 0000000..fb52b18
--- /dev/null
+++ b/p4Sync.py
@@ -0,0 +1,62 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+# author              : Brian Ernst
+# python_version      : 2.7.6 and 3.4.0
+# =================================
+
+from p4Helper import *
+
+import multiprocessing, subprocess, time, traceback
+
+
+#==============================================================
+class P4Sync:
+    def run( self, args ):
+        start = time.clock()
+
+        fail_if_no_p4()
+
+         #http://docs.python.org/library/optparse.html
+        parser = optparse.OptionParser( )
+
+        parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
+        parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=0 )
+        parser.add_option( "-f", "--force", action="store_true", dest="force", help="Force sync files, even if you already have them.", default=False )
+        parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
+        parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
+
+        ( options, args ) = parser.parse_args( args )
+
+        directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
+        thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + options.thread_count
+
+        with Console( auto_flush_time=1 ) as c:
+            with P4Workspace( directory ):
+                if not options.quiet:
+                    c.writeflush( "Syncing files..." )
+                try:
+                    # in progress, very ugly right now.
+                    cmd = "p4 " + \
+                        ( "-vnet.maxwait=60 " if thread_count > 1 else '' ) + \
+                        "-r 100000 sync " + \
+                        ('-f ' if options.force else '') + \
+                        ("--parallel=threads=" + str(thread_count) + " " if thread_count > 1 else '') + \
+                        os.path.join(directory, "...")
+                    c.writeflush(thread_count)
+                    c.writeflush(cmd)
+                    subprocess.check_output( cmd, shell=True )
+                except subprocess.CalledProcessError:
+                    pass
+        if not options.quiet:
+            end = time.clock()
+            delta = end - start
+            output = " Done. Finished in " + str(delta) + "s"
+            print( output )
+
+
+if __name__ == "__main__":
+    try:
+        P4Sync().run(sys.argv)
+    except:
+        print( "\nUnexpected error!" )
+        traceback.print_exc( file = sys.stdout )
\ No newline at end of file
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index 15d418a..76e7ad0 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -26,7 +26,6 @@ class P4SyncMissing:
         parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=12 )
         parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
         parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
-        parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
 
         ( options, args ) = parser.parse_args( args )
 
@@ -45,7 +44,7 @@ class P4SyncMissing:
                     return False
                 def sync( files ):
                     files_len = len(files)
-                    files_flat = ' '.join('"' + f + '"' for f in files)
+                    files_flat = ' '.join('"' + p4FriendlyPath( f ) + '"' for f in files)
 
                     if options.verbose:
                         files_len = len(files)
@@ -82,7 +81,7 @@ class P4SyncMissing:
                 }
 
                 threads = [ ]
-                thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
+                thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + options.thread_count
 
                 count = 0
                 total = 0

From 55e50337942f7efac9a56cf5839f85b0ee2a0d1f Mon Sep 17 00:00:00 2001
From: leetNightshade 
Date: Mon, 8 Jun 2015 21:20:28 -0600
Subject: [PATCH 27/32] Fixed error, forgot to comment out a line.

---
 p4Helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/p4Helper.py b/p4Helper.py
index b11d768..17546d1 100644
--- a/p4Helper.py
+++ b/p4Helper.py
@@ -270,7 +270,7 @@ class P4Workspace:
     def __exit__( self, type, value, tb ):
         # If we changed the current workspace, switch it back.
         if self.oldworkspace_name is not None:
-            c.write("\nReverting back to original client view...")
+            #c.write("\nReverting back to original client view...")
             # set workspace back to the original one
             if try_call_process( 'p4 set P4CLIENT=' + self.oldworkspace_name ):
             #    error_count += 1 # have console log errors

From 972e9ca689d96dbc94a548b2d982da908cc948f4 Mon Sep 17 00:00:00 2001
From: unknown 
Date: Tue, 9 Jun 2015 10:15:00 -0600
Subject: [PATCH 28/32] Fixed comparison issue, apparently had to make sure the
 number was an int. Stupid fucking error.

---
 p4Sync.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/p4Sync.py b/p4Sync.py
index fb52b18..113422f 100644
--- a/p4Sync.py
+++ b/p4Sync.py
@@ -28,7 +28,7 @@ class P4Sync:
         ( options, args ) = parser.parse_args( args )
 
         directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
-        thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + options.thread_count
+        thread_count = int(options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + options.thread_count)
 
         with Console( auto_flush_time=1 ) as c:
             with P4Workspace( directory ):
@@ -42,8 +42,6 @@ class P4Sync:
                         ('-f ' if options.force else '') + \
                         ("--parallel=threads=" + str(thread_count) + " " if thread_count > 1 else '') + \
                         os.path.join(directory, "...")
-                    c.writeflush(thread_count)
-                    c.writeflush(cmd)
                     subprocess.check_output( cmd, shell=True )
                 except subprocess.CalledProcessError:
                     pass

From e5a84235cbe8ece44c902474850313fd0715e252 Mon Sep 17 00:00:00 2001
From: leetNightshade 
Date: Thu, 20 Apr 2017 16:20:43 -0700
Subject: [PATCH 29/32] Fix issue where clientRoot is null due to multiple view
 mappings that don't share one root. TODO: should probably leave getClientRoot
 to return the "null". It's different than returning None.

---
 p4Helper.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/p4Helper.py b/p4Helper.py
index 17546d1..7e1f28b 100644
--- a/p4Helper.py
+++ b/p4Helper.py
@@ -196,7 +196,8 @@ def get_client_root( ):
             continue
 
         local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
-
+        if local_path == "null":
+            local_path = None
         return local_path
     return None
 
@@ -230,22 +231,33 @@ class P4Workspace:
         ldirectory = self.directory.lower()
         oldworkspace_name = None
 
+        # If workspace root is null, it could be because there are multiple views and not a single root.
+        if client_root is None:
+            results = get_p4_py_results('where', self.directory)
+            for result in results:
+                path = result[b'path']
+                path = re.sub('...$', '', path)
+                path = normpath(path)
+                if ldirectory.startswith(path.lower()):
+                    client_root = path
+                    break
+
         if client_root is None or not ldirectory.startswith(client_root.lower()):
             #print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...")
 
             oldworkspace_name = parse_info_from_command('p4 info', 'Client name: ')
 
             # get user workspaces
-            result = get_p4_py_results('workspaces -u ' + username)
+            results = get_p4_py_results('workspaces -u ' + username)
             workspaces = []
-            for r in result:
+            for r in results:
                 whost = get_str_from_process_stdout(r[b'Host'])
                 if whost is not None and len(whost) != 0 and client_host != whost:
                     continue
                 workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])}
                 workspaces.append(workspace)
 
-            del result
+            del results
 
             # check current directory against current workspace, see if it matches existing workspaces.
             for w in workspaces:

From 7f3c4b1cb89bec7a5cc4bd6b61b25c98f739ab70 Mon Sep 17 00:00:00 2001
From: Brian Ernst 
Date: Wed, 12 Aug 2020 14:49:57 -0700
Subject: [PATCH 30/32] Updated time.clock to time.time, since former was
 deprecated.

---
 p4RemoveUnversioned.py | 4 ++--
 p4Sync.py              | 4 ++--
 p4SyncMissingFiles.py  | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py
index eaaa960..ea9cbf2 100644
--- a/p4RemoveUnversioned.py
+++ b/p4RemoveUnversioned.py
@@ -18,7 +18,7 @@ import time, traceback
 
 #==============================================================
 def main( args ):
-    start = time.clock()
+    start = time.time()
 
     fail_if_no_p4()
 
@@ -143,7 +143,7 @@ def main( args ):
                 if error_count > 0:
                     output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )
 
-                end = time.clock()
+                end = time.time()
                 delta = end - start
                 output += "\nFinished in " + str(delta) + "s"
 
diff --git a/p4Sync.py b/p4Sync.py
index 113422f..80d3e9f 100644
--- a/p4Sync.py
+++ b/p4Sync.py
@@ -12,7 +12,7 @@ import multiprocessing, subprocess, time, traceback
 #==============================================================
 class P4Sync:
     def run( self, args ):
-        start = time.clock()
+        start = time.time()
 
         fail_if_no_p4()
 
@@ -46,7 +46,7 @@ class P4Sync:
                 except subprocess.CalledProcessError:
                     pass
         if not options.quiet:
-            end = time.clock()
+            end = time.time()
             delta = end - start
             output = " Done. Finished in " + str(delta) + "s"
             print( output )
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index 76e7ad0..1eecdcd 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -15,7 +15,7 @@ import time, traceback
 #==============================================================
 class P4SyncMissing:
     def run( self, args ):
-        start = time.clock()
+        start = time.time()
 
         fail_if_no_p4()
 
@@ -205,7 +205,7 @@ class P4SyncMissing:
                     print( "  Done." )
 
         if not options.quiet:
-            end = time.clock()
+            end = time.time()
             delta = end - start
             output = " Done. Finished in " + str(delta) + "s"
             print( output )

From 3adaf471c1354be6682f50c978a19b4956ae769f Mon Sep 17 00:00:00 2001
From: Brian Ernst 
Date: Wed, 12 Aug 2020 14:55:20 -0700
Subject: [PATCH 31/32] Updated readme.

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5942e43..20a3620 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,9 @@
 p4Tools
 ===================
 
-Removes unversioned files from perforce repository. Script is in beta, works well, but still going through continued testing.  There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required.
+Removes unversioned files from perforce repository. 
+
+Script is in beta, works well, but still going through continued testing.  There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required. One of the reasons this is still in testing is because sometimes the end of the script gets stuck when closing Console logging; I haven't had the time or care to fix this, so it's not considered stable or production ready for at least that reason.
 
 Concerning benchmarks: I used to have a HDD, now a SSD.  So I can't provide valid comparisons to the old numbers until I do them on a computer with a HDD.  That said, this single worker implementation runs faster than the old multi-threaded version.  Can't wait to further update it, will only continue to get faster.
 

From 3aa137375897a50a848242e2a7b1782bb0e2e079 Mon Sep 17 00:00:00 2001
From: Brian Ernst 
Date: Wed, 12 Aug 2020 15:05:50 -0700
Subject: [PATCH 32/32] Minor cleanup.

---
 README.md              | 6 +++++-
 p4RemoveUnversioned.py | 7 +++----
 p4SyncMissingFiles.py  | 5 +++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 20a3620..1f22b4f 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,11 @@
 p4Tools
 ===================
 
-Removes unversioned files from perforce repository. 
+Perforce script tools for:
+* Remove unversioned files
+* Parallel sync missing files
+* Parallel sync everything
+* Etc.
 
 Script is in beta, works well, but still going through continued testing.  There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required. One of the reasons this is still in testing is because sometimes the end of the script gets stuck when closing Console logging; I haven't had the time or care to fix this, so it's not considered stable or production ready for at least that reason.
 
diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py
index ea9cbf2..da3d3a1 100644
--- a/p4RemoveUnversioned.py
+++ b/p4RemoveUnversioned.py
@@ -4,11 +4,9 @@
 # python_version      : 2.7.6 and 3.4.0
 # =================================
 
-# todo: switch to `p4 fstat ...`, and parse the output for clientFile and cache it.
 # todo: have a backup feature, make sure files are moved to the recycle bin or a temporary file.
-# todo: switch to faster method of calling p4 fstat on an entire directory and parsing it's output
 # todo: add option of using send2trash
-# todo: buffer output, after exceeding a certain amount print to the output.
+# todo: switch to faster method of calling p4 fstat on an entire directory and parsing it's output
 # todo: allow logging output besides console output, or redirection altogether
 
 from p4Helper import *
@@ -55,7 +53,8 @@ def main( args ):
             # TODO: push this off to a thread and walk the directory so we get a headstart.
             files_in_depot = get_client_set( directory )
 
-            c.writeflush( "|Done." )
+            if not options.quiet:
+                c.writeflush( "|Done." )
 
             # TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?
 
diff --git a/p4SyncMissingFiles.py b/p4SyncMissingFiles.py
index 1eecdcd..ca733c9 100644
--- a/p4SyncMissingFiles.py
+++ b/p4SyncMissingFiles.py
@@ -19,7 +19,7 @@ class P4SyncMissing:
 
         fail_if_no_p4()
 
-         #http://docs.python.org/library/optparse.html
+        #http://docs.python.org/library/optparse.html
         parser = optparse.OptionParser( )
 
         parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
@@ -93,7 +93,8 @@ class P4SyncMissing:
                     threads.append( t )
                     t.start( )
 
-                c.writeflush( "  Done." )
+                if not options.quiet:
+                    c.writeflush( "  Done." )
 
                 make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False