diff --git a/p4RemoveUnversioned.py b/p4RemoveUnversioned.py index 668dacb..1c61548 100644 --- a/p4RemoveUnversioned.py +++ b/p4RemoveUnversioned.py @@ -77,6 +77,13 @@ def match_in_ignore_list( path, ignore_list ): return True return False +use_bytearray_str_conversion = type( b"str" ) is not str +def get_str_from_process_stdout( line ): + if use_bytearray_str_conversion: + return ''.join( map( chr, line ) ) + else: + return line + # Keep these in mind if you have issues: # https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running @@ -89,6 +96,8 @@ def get_client_set( path ): proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=None, cwd=path ) for line in proc.stdout: + line = get_str_from_process_stdout( line ) + clientFile_tag = "... clientFile " if not line.startswith( clientFile_tag ): continue @@ -112,6 +121,7 @@ class PDict( dict ): dict.__init__( self, args ) self.mutex = multiprocessing.Semaphore( ) +# TODO: Create a child thread for triggering autoflush events class Console( threading.Thread ): MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' ) @@ -124,6 +134,7 @@ class Console( threading.Thread ): self.queue = multiprocessing.JoinableQueue( ) self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1 self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1 + self.shutting_down = False def write( self, data, pid = None ): self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) ) @@ -158,7 +169,11 @@ class Console( threading.Thread ): for line in buffer: print( line ) self.buffers.clear( ) + self.buffer_write_times.clear( ) self.queue.task_done( ) + + print(self.queue.qsize()) + print(self.queue.empty()) break elif event == Console.MSG.WRITE: @@ -174,6 +189,7 @@ class Console( threading.Thread ): self.flush( pid ) elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time: self.flush( pid ) + # TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time elif event == Console.MSG.FLUSH: pid = data[ 1 ] if pid in self.buffers: @@ -188,128 +204,6 @@ class Console( threading.Thread ): self.queue.task_done( ) -class Worker( threading.Thread ): - def __init__( self, console, queue, files_to_ignore ): - threading.Thread.__init__( self ) - - self.console = console - self.queue = queue - self.files_to_ignore = files_to_ignore - - def run( self ): - while True: - ( cmd, data ) = self.queue.get( ) - - if cmd == MSG.SHUTDOWN: - self.queue.task_done( ) - self.console.flush( ) - break - - if cmd != MSG.PARSE_DIRECTORY or data is None: - self.console.flush( ) - self.queue.task_done( ) - continue - - directory = data - - self.console.write( "Working on " + directory ) - - try: - dir_contents = os.listdir( directory ) - except OSError as ex: - self.console.write( "| " + type( ex ).__name__ ) - # args can be anything, can't guarantee they'll convert to a string - #self.console.write( "| " + ' '.join( [ str( arg ) for arg in ex.args ] ) ) - self.console.write( "| " + repr( ex ) ) - self.console.write( "|ERROR." ) - self.console.flush( ) - self.queue.task_done( ) - continue - - if p4_ignore in dir_contents: - file_regexes = [] - # Should automatically ignore .p4ignore even if it's not specified, otherwise it'll be deleted. - path = join( directory, p4_ignore ) - with open( path ) as f: - for line in f: - new_line = remove_comment( line.strip( ) ) - if len( new_line ) > 0: - # doesn't look quite right, fix it: - file_regexes.append( re.compile( join( re.escape( directory + os.sep ), new_line ) ) ) - - self.console.write( "| Appending ignores from " + path ) - with self.files_to_ignore.mutex: - if directory not in self.files_to_ignore: - self.files_to_ignore[ directory ] = [] - self.files_to_ignore[ directory ].extend( file_regexes ) - - - ignore_list = get_ignore_list( directory, self.files_to_ignore ) - - - files = [] - command = "p4 fstat *" - - try: - proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory ) - (out, err) = proc.communicate() - except Exception as ex: - self.console.write( "| " + type( ex ).__name__ ) - # args can be anything, can't guarantee they'll convert to a string - #self.console.write( "| " + ' '.join( [ str( arg ) for arg in ex.args ] ) ) - self.console.write( "| " + repr( ex ) ) - self.console.write( "|ERROR." ) - self.console.flush( ) - self.queue.task_done( ) - continue - - for line in err.decode('utf-8').split( os.linesep ): - if len( line ) == 0: - continue - - # # dirty hack that grabs the filename from the ends of the printed out (not err) "depo_path - local_path" - # # I could use regex to verify the expected string, but that will just slow us down. - # base = basename( line ) - i = line.rfind( ' - ') - if i >= 0: - base = line[ : i ] - if base == "*" or len(base) == 0: - # Directory is empty, we could delete it now - continue - path = join( directory, base ) - - if not os.path.isdir( path ): - files.append( base ) - - for content in dir_contents: - path = join( directory, content ) - if os.path.isdir( path ): - if match_in_ignore_list( path, ignore_list ): - self.console.write( "| Ignoring " + content ) - else: - self.queue.put( ( MSG.PARSE_DIRECTORY, path ) ) - - for file in files: - path = join( directory, file ) - - if match_in_ignore_list( path, ignore_list ): - self.console.write( "| Ignoring " + path ) - continue - - self.console.write( "| " + file + " is unversioned, removing it." ) - try: - os.chmod( path, stat.S_IWRITE ) - os.remove( path ) - except OSError as ex: - self.console.write( "| " + type( ex ).__name__ ) - self.console.write( "| " + repr( ex ) ) - self.console.write( "|ERROR." ) - - self.console.write( "|Done." ) - self.console.flush( ) - - self.queue.task_done( ) - def main( args ): # check requirements if os.system( 'p4 > Nul' ) != 0: @@ -334,7 +228,7 @@ def main( args ): c.writeflush( "Checking " + directory) for root, dirs, files in os.walk( directory ): - ignore_list = PDict()#get_ignore_list( root, files_to_ignore ) + ignore_list = {}#get_ignore_list( root, files_to_ignore ) c.write( "|Checking " + root ) @@ -359,6 +253,7 @@ def main( args ): # c.writeflush( "| " + type( ex ).__name__ ) # c.writeflush( "| " + repr( ex ) ) # c.writeflush( "|ERROR." ) + c.write( "|Done." ) if __name__ == "__main__":