Split scripts up for now, may add all in one scripts later. Added

p4SyncMissingFiles.py, so you don't have to do a force sync and redownload
everything.
This commit is contained in:
unknown 2015-02-18 15:09:57 -07:00
parent 9d4d26250d
commit 1d1d7f8cae
5 changed files with 648 additions and 476 deletions

View File

@ -1,4 +1,4 @@
p4RemoveUnversioned
p4Tools
===================
Removes unversioned files from perforce repository. Script is in beta, works well, but still going through continued testing. There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required.
@ -6,6 +6,6 @@ Removes unversioned files from perforce repository. Script is in beta, works wel
Concerning benchmarks: I used to have a HDD, now a SSD. So I can't provide valid comparisons to the old numbers until I do them on a computer with a HDD. That said, this single worker implementation runs faster than the old multi-threaded version. Can't wait to further update it, will only continue to get faster.
This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex.
~~This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex.~~ I need to re-add this to the newer script.
**Files are currently permanently deleted, so use this at your own risk.**

398
p4Helper.py Normal file
View File

@ -0,0 +1,398 @@
#!/usr/bin/python
# -*- coding: utf8 -*-
# author : Brian Ernst
# python_version : 2.7.6 and 3.4.0
# =================================
import datetime, inspect, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading
# trying ntpath, need to test on linux
import ntpath
try: input = raw_input
except: pass
#==============================================================
re_remove_comment = re.compile( "#.*$" )
def remove_comment( s ):
return re.sub( re_remove_comment, "", s )
def singular_pulural( val, singular, plural ):
return singular if val == 1 else plural
#==============================================================
def enum(*sequential, **named):
enums = dict(zip(sequential, range(len(sequential))), **named)
return type('Enum', (), enums)
MSG = enum('SHUTDOWN', 'PARSE_DIRECTORY', 'RUN_FUNCTION')
p4_ignore = ".p4ignore"
main_pid = os.getpid( )
#==============================================================
#if os.name == 'nt' or sys.platform == 'cygwin'
def basename( path ):
# TODO: import based on platform
# https://docs.python.org/2/library/os.path.html
# posixpath for UNIX-style paths
# ntpath for Windows paths
# macpath for old-style MacOS paths
# os2emxpath for OS/2 EMX paths
#return os.path.basename( path )
return ntpath.basename( path )
def normpath( path ):
return ntpath.normpath( path )
def join( patha, pathb ):
return ntpath.join( patha, pathb )
def splitdrive( path ):
return ntpath.splitdrive( path )
#==============================================================
def get_ignore_list( path, files_to_ignore ):
# have to split path and test top directory
dirs = path.split( os.sep )
ignore_list = [ ]
for i, val in enumerate( dirs ):
path_to_find = os.sep.join( dirs[ : i + 1] )
if path_to_find in files_to_ignore:
ignore_list.extend( files_to_ignore[ path_to_find ] )
return ignore_list
def match_in_ignore_list( path, ignore_list ):
for r in ignore_list:
if re.match( r, path ):
return True
return False
#==============================================================
def call_process( args ):
return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
def try_call_process( args, path=None ):
try:
subprocess.check_output( args.split( ), shell=False, cwd=path )
return 0
except subprocess.CalledProcessError:
return 1
use_bytearray_str_conversion = type( b"str" ) is not str
def get_str_from_process_stdout( line ):
if use_bytearray_str_conversion:
return ''.join( map( chr, line ) )
else:
return line
def parse_info_from_command( args, value, path = None ):
"""
:rtype : string
"""
proc = subprocess.Popen( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
if not line.startswith( value ):
continue
return line[ len( value ) : ].strip( )
return None
def get_p4_py_results( args, path = None ):
results = []
proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
try:
while True:
output = marshal.load( proc.stdout )
results.append( output )
except EOFError:
pass
finally:
proc.stdout.close()
return results
#==============================================================
def fail_if_no_p4():
if call_process( 'p4 -V' ) != 0:
print( 'Perforce Command-line Client(p4) is required for this script.' )
sys.exit( 1 )
# Keep these in mind if you have issues:
# https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime
# https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
def get_client_set( path ):
files = set( [ ] )
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
command = "p4 fstat ..."
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "... clientFile "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
if make_drive_upper:
drive, path = splitdrive( local_path )
local_path = ''.join( [ drive.upper( ), path ] )
files.add( local_path )
proc.wait( )
for line in proc.stderr:
if "no such file" in line:
continue
raise Exception(line)
return files
def get_client_root( ):
"""
:rtype : string
"""
command = "p4 info"
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "Client root: "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
return local_path
return None
class P4Workspace:
"""
Use this class when working in a workspace.
Makes sure the environmentals are setup correctly, and that you aren't working on a non-perforce directory accidentally;
otherwise you can delete files that shouldn't be deleted. Ex:
with P4Workspace( cwd ): #sets current workspace to cwd, or fails
# do stuff here
# on exit reverts to previous set workspace
"""
def __init__( self, directory):
self.directory = directory
def __enter__( self ):
# get user
#print("\nChecking p4 info...")
result = get_p4_py_results('info')
if len(result) == 0 or b'userName' not in result[0].keys():
print("Can't find perforce info, is it even setup?")
sys.exit(1)
username = get_str_from_process_stdout(result[0][b'userName'])
client_host = get_str_from_process_stdout(result[0][b'clientHost'])
#print("|Done.")
# see if current directory is set to current workspace, if not, set it to current workspace.
client_root = get_client_root()
ldirectory = self.directory.lower()
oldworkspace_name = None
if client_root is None or not ldirectory.startswith(client_root.lower()):
#print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...")
oldworkspace_name = parse_info_from_command('p4 info', 'Client name: ')
# get user workspaces
result = get_p4_py_results('workspaces -u ' + username)
workspaces = []
for r in result:
whost = get_str_from_process_stdout(r[b'Host'])
if whost is not None and len(whost) != 0 and client_host != whost:
continue
workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])}
workspaces.append(workspace)
del result
# check current directory against current workspace, see if it matches existing workspaces.
for w in workspaces:
wname = w['name']
wlower = w['root'].lower()
if ldirectory.startswith(wlower):
# set current directory, don't forget to revert it back to the existing one
#print("|Setting client view to: " + wname)
if try_call_process( 'p4 set P4CLIENT=' + wname ):
#print("|There was a problem trying to set the p4 client view (workspace).")
sys.exit(1)
break
else:
# TODO: look up workspace/users for this computer
print( "Couldn't find a workspace root that matches the current directory for the current user." )
sys.exit(1)
#print("|Done.")
self.oldworkspace_name = oldworkspace_name
return self
def __exit__( self, type, value, tb ):
# If we changed the current workspace, switch it back.
if self.oldworkspace_name is not None:
c.write("\nReverting back to original client view...")
# set workspace back to the original one
if try_call_process( 'p4 set P4CLIENT=' + self.oldworkspace_name ):
# error_count += 1 # have console log errors
# if not options.quiet:
print("There was a problem trying to restore the set p4 client view (workspace).")
sys.exit(1)
#else:
# if not options.quiet:
# c.write("|Reverted client view back to '" + self.oldworkspace_name + "'.")
#if not options.quiet:
# c.write("|Done.")
#==============================================================
class PTable( list ):
def __init__( self, *args ):
list.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
class PDict( dict ):
def __init__( self, *args ):
dict.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
#==============================================================
# TODO: Create a child thread for triggering autoflush events
# TODO: Hook console into stdout so it catches print
class Console( threading.Thread ):
MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' )
# auto_flush_time is time in milliseconds since last flush to trigger a flush when writing
def __init__( self, auto_flush_num = None, auto_flush_time = None ):
threading.Thread.__init__( self )
self.buffers = {}
self.buffer_write_times = {}
self.running = True
self.queue = multiprocessing.JoinableQueue( )
self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1
self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
self.shutting_down = False
def write( self, data, pid = None ):
self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) )
def writeflush( self, data, pid = None ):
pid = pid if pid is not None else os.getpid()
self.queue.put( ( Console.MSG.WRITE, pid, data ) )
self.queue.put( ( Console.MSG.FLUSH, pid ) )
def flush( self, pid = None ):
self.queue.put( ( Console.MSG.FLUSH, pid if pid is not None else os.getpid() ) )
def clear( self, pid = None ):
self.queue.put( ( Console.MSG.CLEAR, pid if pid is not None else os.getpid() ) )
def __enter__( self ):
self.start( )
return self
def __exit__( self, type, value, tb ):
self.queue.put( ( Console.MSG.SHUTDOWN, ) )
self.queue.join( )
def run( self ):
while True:
data = self.queue.get( )
event = data[0]
if event == Console.MSG.SHUTDOWN:
# flush remaining buffers before shutting down
for ( pid, buffer ) in self.buffers.items( ):
for line in buffer:
print( line )
self.buffers.clear( )
self.buffer_write_times.clear( )
self.queue.task_done( )
#print(self.queue.qsize())
#print(self.queue.empty())
break
elif event == Console.MSG.WRITE:
pid, s = data[ 1 : ]
if pid not in self.buffers:
self.buffers[ pid ] = []
if pid not in self.buffer_write_times:
self.buffer_write_times[ pid ] = datetime.datetime.now( )
self.buffers[ pid ].append( s )
if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num:
self.flush( pid )
elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time:
self.flush( pid )
# TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time
elif event == Console.MSG.FLUSH:
pid = data[ 1 ]
if pid in self.buffers:
for line in self.buffers[ pid ]:
print( line )
self.buffers.pop( pid, None )
self.buffer_write_times[ pid ] = datetime.datetime.now( )
elif event == Console.MSG.CLEAR:
pid = data[ 1 ]
if pid in self.buffers:
self.buffers.pop( pid, None )
self.queue.task_done( )
#==============================================================
# class Task( threading.Event ):
# def __init__( data, cmd = None ):
# threading.Event.__init__( self )
# self.cmd = cmd if cmd is None MSG.RUN_FUNCTION
# self.data = data
# def isDone( self ):
# return self.isSet()
# def join( self ):
# self.wait( )
#==============================================================
class Worker( threading.Thread ):
def __init__( self, console, queue, commands ):
threading.Thread.__init__( self )
self.queue = queue
self.commands = commands
def run( self ):
while True:
( cmd, data ) = self.queue.get( )
if not self.commands[cmd](data):
self.queue.task_done( )
break
self.queue.task_done( )

BIN
p4Helper.pyc Normal file

Binary file not shown.

View File

@ -11,317 +11,16 @@
# todo: buffer output, after exceeding a certain amount print to the output.
# todo: allow logging output besides console output, or redirection altogether
import datetime, inspect, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading, time, traceback
from p4Helper import *
# trying ntpath, need to test on linux
import ntpath
import time, traceback
re_remove_comment = re.compile( "#.*$" )
def remove_comment( s ):
return re.sub( re_remove_comment, "", s )
try: input = raw_input
except: pass
def enum(*sequential, **named):
enums = dict(zip(sequential, range(len(sequential))), **named)
return type('Enum', (), enums)
MSG = enum('SHUTDOWN', 'PARSE_DIRECTORY', 'RUN_FUNCTION')
p4_ignore = ".p4ignore"
main_pid = os.getpid( )
#if os.name == 'nt' or sys.platform == 'cygwin'
def basename( path ):
# TODO: import based on platform
# https://docs.python.org/2/library/os.path.html
# posixpath for UNIX-style paths
# ntpath for Windows paths
# macpath for old-style MacOS paths
# os2emxpath for OS/2 EMX paths
#return os.path.basename( path )
return ntpath.basename( path )
def normpath( path ):
return ntpath.normpath( path )
def join( patha, pathb ):
return ntpath.join( patha, pathb )
def splitdrive( path ):
return ntpath.splitdrive( path )
def get_ignore_list( path, files_to_ignore ):
# have to split path and test top directory
dirs = path.split( os.sep )
ignore_list = [ ]
for i, val in enumerate( dirs ):
path_to_find = os.sep.join( dirs[ : i + 1] )
if path_to_find in files_to_ignore:
ignore_list.extend( files_to_ignore[ path_to_find ] )
return ignore_list
def match_in_ignore_list( path, ignore_list ):
for r in ignore_list:
if re.match( r, path ):
return True
return False
def call_process( args ):
return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
def try_call_process( args, path=None ):
try:
subprocess.check_output( args.split( ), shell=False, cwd=path )
return 0
except subprocess.CalledProcessError:
return 1
use_bytearray_str_conversion = type( b"str" ) is not str
def get_str_from_process_stdout( line ):
if use_bytearray_str_conversion:
return ''.join( map( chr, line ) )
else:
return line
def singular_pulural( val, singular, plural ):
return singular if val == 1 else plural
def parse_info_from_command( args, value, path = None ):
"""
:rtype : string
"""
proc = subprocess.Popen( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
if not line.startswith( value ):
continue
return line[ len( value ) : ].strip( )
return None
def get_p4_py_results( args, path = None ):
results = []
proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
try:
while True:
output = marshal.load( proc.stdout )
results.append( output )
except EOFError:
pass
finally:
proc.stdout.close()
return results
# Keep these in mind if you have issues:
# https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime
# https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
def get_client_set( path ):
files = set( [ ] )
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
command = "p4 fstat ..."
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "... clientFile "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
if make_drive_upper:
drive, path = splitdrive( local_path )
local_path = ''.join( [ drive.upper( ), path ] )
files.add( local_path )
proc.wait( )
for line in proc.stderr:
if "no such file" in line:
continue
raise Exception(line)
return files
def get_client_root( ):
"""
:rtype : string
"""
command = "p4 info"
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "Client root: "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
return local_path
return None
class PTable( list ):
def __init__( self, *args ):
list.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
class PDict( dict ):
def __init__( self, *args ):
dict.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
# TODO: Create a child thread for triggering autoflush events
class Console( threading.Thread ):
MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' )
# auto_flush_time is time in milliseconds since last flush to trigger a flush when writing
def __init__( self, auto_flush_num = None, auto_flush_time = None ):
threading.Thread.__init__( self )
self.buffers = {}
self.buffer_write_times = {}
self.running = True
self.queue = multiprocessing.JoinableQueue( )
self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1
self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
self.shutting_down = False
def write( self, data, pid = None ):
self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) )
def writeflush( self, data, pid = None ):
pid = pid if pid is not None else os.getpid()
self.queue.put( ( Console.MSG.WRITE, pid, data ) )
self.queue.put( ( Console.MSG.FLUSH, pid ) )
def flush( self, pid = None ):
self.queue.put( ( Console.MSG.FLUSH, pid if pid is not None else os.getpid() ) )
def clear( self, pid = None ):
self.queue.put( ( Console.MSG.CLEAR, pid if pid is not None else os.getpid() ) )
def __enter__( self ):
self.start( )
return self
def __exit__( self, type, value, tb ):
self.queue.put( ( Console.MSG.SHUTDOWN, ) )
self.queue.join( )
def run( self ):
while True:
data = self.queue.get( )
event = data[0]
if event == Console.MSG.SHUTDOWN:
# flush remaining buffers before shutting down
for ( pid, buffer ) in self.buffers.items( ):
for line in buffer:
print( line )
self.buffers.clear( )
self.buffer_write_times.clear( )
self.queue.task_done( )
#print(self.queue.qsize())
#print(self.queue.empty())
break
elif event == Console.MSG.WRITE:
pid, s = data[ 1 : ]
if pid not in self.buffers:
self.buffers[ pid ] = []
if pid not in self.buffer_write_times:
self.buffer_write_times[ pid ] = datetime.datetime.now( )
self.buffers[ pid ].append( s )
if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num:
self.flush( pid )
elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time:
self.flush( pid )
# TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time
elif event == Console.MSG.FLUSH:
pid = data[ 1 ]
if pid in self.buffers:
for line in self.buffers[ pid ]:
print( line )
self.buffers.pop( pid, None )
self.buffer_write_times[ pid ] = datetime.datetime.now( )
elif event == Console.MSG.CLEAR:
pid = data[ 1 ]
if pid in self.buffers:
self.buffers.pop( pid, None )
self.queue.task_done( )
# class Task( threading.Event ):
# def __init__( data, cmd = None ):
# threading.Event.__init__( self )
# self.cmd = cmd if cmd is None MSG.RUN_FUNCTION
# self.data = data
# def isDone( self ):
# return self.isSet()
# def join( self ):
# self.wait( )
class Worker( threading.Thread ):
def __init__( self, console, queue, files_to_ignore ):
threading.Thread.__init__( self )
self.console = console
self.queue = queue
self.files_to_ignore = files_to_ignore
def run( self ):
while True:
( cmd, data ) = self.queue.get( )
if cmd == MSG.SHUTDOWN:
self.console.flush( )
self.queue.task_done( )
break
if cmd == MSG.RUN_FUNCTION:
break
if cmd != MSG.PARSE_DIRECTORY or data is None:
self.console.flush( )
self.queue.task_done( )
continue
directory = data
# add threading stuffs
self.queue.task_done( )
#==============================================================
def main( args ):
start = time.clock()
# check requirements
if call_process( 'p4 -V' ) != 0:
print( 'Perforce Command-line Client(p4) is required for this script.' )
sys.exit( 1 )
fail_if_no_p4()
#http://docs.python.org/library/optparse.html
parser = optparse.OptionParser( )
@ -336,184 +35,123 @@ def main( args ):
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
# get user
print("\nChecking p4 info...")
result = get_p4_py_results('info')
if len(result) == 0 or b'userName' not in result[0].keys():
print("Can't find perforce info, is it even setup?")
sys.exit(1)
username = get_str_from_process_stdout(result[0][b'userName'])
client_host = get_str_from_process_stdout(result[0][b'clientHost'])
print("|Done.")
client_root = get_client_root()
ldirectory = directory.lower()
workspace_name = None
if client_root is None or not ldirectory.startswith(client_root.lower()):
print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...")
workspace_name = parse_info_from_command('p4 info', 'Client name: ')
# get user workspaces
result = get_p4_py_results('workspaces -u ' + username)
workspaces = []
for r in result:
whost = get_str_from_process_stdout(r[b'Host'])
if whost is not None and len(whost) != 0 and client_host != whost:
continue
workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])}
workspaces.append(workspace)
del result
# check current directory against current workspace, see if it matches existing workspaces.
for w in workspaces:
wname = w['name']
wlower = w['root'].lower()
if ldirectory.startswith(wlower):
# set current directory, don't forget to revert it back to the existing one
print("|Setting client view to: " + wname)
if try_call_process( 'p4 set P4CLIENT=' + wname ):
print("|There was a problem trying to set the p4 client view (workspace).")
sys.exit(1)
break
else:
print( "|Couldn't find a workspace root that matches the current directory for the current user." )
sys.exit(1)
print("|Done.")
# Files are added from .p4ignore
# Key is the file root, the value is the table of file regexes for that directory.
files_to_ignore = PDict()
processed_file_count = 0
processed_directory_count = 0
remove_file_count = 0
remove_dir_count = 0
warning_count = 0
error_count = 0
with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
if not options.quiet:
c.writeflush( "\nCaching files in depot, this may take a little while..." )
with P4Workspace( directory ):
# Files are added from .p4ignore
# Key is the file root, the value is the table of file regexes for that directory.
files_to_ignore = PDict()
# TODO: push this off to a thread and walk the directory so we get a headstart.
files_in_depot = get_client_set( directory )
c.writeflush( "|Done." )
# TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?
# TODO: at this point join on both tasks to wait until they're done
# TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
# need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
# will at least need to redo how the ignore lists are handled for efficiencies sake.
if not options.quiet:
c.writeflush( "\nChecking " + directory)
for root, dirs, files in os.walk( directory ):
ignore_list = get_ignore_list( root, files_to_ignore )
if not options.quiet:
c.write( "|Checking " + os.path.relpath( root, directory ) )
for d in dirs:
processed_directory_count += 1
path = join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ):
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
for f in files:
processed_file_count += 1
path = normpath( join( root, f ) )
if path not in files_in_depot:
if not options.quiet:
c.write( "| " + f + " is unversioned, removing it." )
try:
os.chmod( path, stat.S_IWRITE )
os.remove( path )
remove_file_count += 1
except OSError as ex:
c.writeflush( "| " + type( ex ).__name__ )
c.writeflush( "| " + repr( ex ) )
c.writeflush( "| ^ERROR^" )
error_count += 1
if not options.quiet:
c.write( "|Done." )
if not options.quiet:
c.write( os.linesep + "Removing empty directories...")
# remove empty directories in reverse order
for root, dirs, files in os.walk( directory, topdown=False ):
ignore_list = get_ignore_list( root, files_to_ignore )
for d in dirs:
processed_directory_count += 1
path = os.path.join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ):
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
try:
os.rmdir(path)
remove_dir_count += 1
if not options.quiet:
c.write( "| " + rel_path + " was removed." )
except OSError:
# Fails on non-empty directory
pass
if not options.quiet:
c.write( "|Done." )
# This needs to happen automatically even when an exception happens, when we leave scope.
if workspace_name is not None:
c.write("\nReverting back to original client view...")
# set workspace back to the original one
if try_call_process( 'p4 set P4CLIENT=' + workspace_name ):
error_count += 1
if not options.quiet:
c.write("|There was a problem trying to restore the set p4 client view (workspace).")
else:
if not options.quiet:
c.write("|Reverted client view back to '" + workspace_name + "'.")
if not options.quiet:
c.write("|Done.")
if not options.quiet:
output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " )
output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories")
processed_file_count = 0
processed_directory_count = 0
output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " )
output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories")
remove_file_count = 0
remove_dir_count = 0
warning_count = 0
error_count = 0
if warning_count > 0:
output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" )
if error_count > 0:
output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )
if not options.quiet:
c.writeflush( "\nCaching files in depot, this may take a little while..." )
end = time.clock()
delta = end - start
output += "\nFinished in " + str(delta) + "s"
# TODO: push this off to a thread and walk the directory so we get a headstart.
files_in_depot = get_client_set( directory )
c.write( output )
c.writeflush( "|Done." )
# TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?
# TODO: at this point join on both tasks to wait until they're done
# TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
# need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
# will at least need to redo how the ignore lists are handled for efficiencies sake.
if not options.quiet:
c.writeflush( "\nChecking " + directory)
for root, dirs, files in os.walk( directory ):
ignore_list = get_ignore_list( root, files_to_ignore )
if not options.quiet:
c.write( "|Checking " + os.path.relpath( root, directory ) )
for d in dirs:
processed_directory_count += 1
path = join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ):
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
for f in files:
processed_file_count += 1
path = normpath( join( root, f ) )
if path not in files_in_depot:
if not options.quiet:
c.write( "| " + f + " is unversioned, removing it." )
try:
os.chmod( path, stat.S_IWRITE )
os.remove( path )
remove_file_count += 1
except OSError as ex:
c.writeflush( "| " + type( ex ).__name__ )
c.writeflush( "| " + repr( ex ) )
c.writeflush( "| ^ERROR^" )
error_count += 1
if not options.quiet:
c.write( "|Done." )
if not options.quiet:
c.write( os.linesep + "Removing empty directories...")
# remove empty directories in reverse order
for root, dirs, files in os.walk( directory, topdown=False ):
ignore_list = get_ignore_list( root, files_to_ignore )
for d in dirs:
processed_directory_count += 1
path = os.path.join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ):
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
try:
os.rmdir(path)
remove_dir_count += 1
if not options.quiet:
c.write( "| " + rel_path + " was removed." )
except OSError:
# Fails on non-empty directory
pass
if not options.quiet:
c.write( "|Done." )
if not options.quiet:
output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " )
output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories")
output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " )
output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories")
if warning_count > 0:
output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" )
if error_count > 0:
output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )
end = time.clock()
delta = end - start
output += "\nFinished in " + str(delta) + "s"
c.write( output )
if __name__ == "__main__":
try:
main( sys.argv )
except:
print( "Unexpected error!" )
print( "\nUnexpected error!" )
traceback.print_exc( file = sys.stdout )

136
p4SyncMissingFiles.py Normal file
View File

@ -0,0 +1,136 @@
#!/usr/bin/python
# -*- coding: utf8 -*-
# author : Brian Ernst
# python_version : 2.7.6 and 3.4.0
# =================================
# TODO: setup batches before pushing to threads and use p4 --parallel
# http://www.perforce.com/perforce/r14.2/manuals/cmdref/p4_sync.html
from p4Helper import *
import time, traceback
#==============================================================
def main( args ):
start = time.clock()
fail_if_no_p4()
#http://docs.python.org/library/optparse.html
parser = optparse.OptionParser( )
parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 )
parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
( options, args ) = parser.parse_args( args )
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
with P4Workspace( directory ):
if not options.quiet:
c.writeflush( "Preparing to sync missing files..." )
c.write( " Setting up threads..." )
# Setup threading
WRK = enum( 'SHUTDOWN', 'SYNC' )
def shutdown( data ):
return False
def sync( data ):
if data is not None and not os.path.exists( data ):
try_call_process( "p4 sync -f " + data )
if not options.quiet:
c.write( " Synced " + data )
return True
commands = {
WRK.SHUTDOWN : shutdown,
WRK.SYNC : sync
}
threads = [ ]
thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
queue = multiprocessing.JoinableQueue( )
for i in range( thread_count ):
t = Worker( c, queue, commands )
threads.append( t )
t.start( )
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
command = "p4 fstat ..."
if not options.quiet:
c.writeflush( " Checking files in depot, this may take some time for large depots..." )
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory )
clientFile_tag = "... clientFile "
headAction_tag = "... headAction "
# http://www.perforce.com/perforce/r12.1/manuals/cmdref/fstat.html
accepted_actions = [ 'add', 'edit', 'branch', 'move/add', 'move\\add', 'integrate', 'import', 'archive' ] #currently not checked
rejected_actions = [ 'delete', 'move/delete', 'move\\delete', 'purge' ]
client_file = None
for line in proc.stdout:
line = get_str_from_process_stdout( line )
if client_file and line.startswith( headAction_tag ):
action = normpath( line[ len( headAction_tag ) : ].strip( ) )
if not any(action == a for a in rejected_actions):
if options.verbose:
c.write( " Checking " + os.path.relpath( local_path, directory ) )
queue.put( ( WRK.SYNC, local_path ) )
if line.startswith( clientFile_tag ):
client_file = None
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
if make_drive_upper:
drive, path = splitdrive( local_path )
client_file = ''.join( [ drive.upper( ), path ] )
if len(line.rstrip()) == 0:
client_file = None
proc.wait( )
for line in proc.stderr:
if "no such file" in line:
continue
#raise Exception(line)
c.write(line)#log as error
if not options.quiet:
c.writeflush( " Pushed work, now waiting for threads..." )
for i in range( thread_count ):
queue.put( ( WRK.SHUTDOWN, None ) )
for t in threads:
t.join( )
if not options.quiet:
c.write( "Done." )
end = time.clock()
delta = end - start
output = "\nFinished in " + str(delta) + "s"
c.writeflush( output )
if __name__ == "__main__":
try:
main( sys.argv )
except:
print( "\nUnexpected error!" )
traceback.print_exc( file = sys.stdout )