Split scripts up for now, may add all in one scripts later. Added

p4SyncMissingFiles.py, so you don't have to do a force sync and redownload
everything.
This commit is contained in:
unknown 2015-02-18 15:09:57 -07:00
parent 9d4d26250d
commit 1d1d7f8cae
5 changed files with 648 additions and 476 deletions

View File

@ -1,4 +1,4 @@
p4RemoveUnversioned p4Tools
=================== ===================
Removes unversioned files from perforce repository. Script is in beta, works well, but still going through continued testing. There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required. Removes unversioned files from perforce repository. Script is in beta, works well, but still going through continued testing. There are a few stats at the end, will be putting in more, like number of files/directories checked, so you have an idea how much work was required.
@ -6,6 +6,6 @@ Removes unversioned files from perforce repository. Script is in beta, works wel
Concerning benchmarks: I used to have a HDD, now a SSD. So I can't provide valid comparisons to the old numbers until I do them on a computer with a HDD. That said, this single worker implementation runs faster than the old multi-threaded version. Can't wait to further update it, will only continue to get faster. Concerning benchmarks: I used to have a HDD, now a SSD. So I can't provide valid comparisons to the old numbers until I do them on a computer with a HDD. That said, this single worker implementation runs faster than the old multi-threaded version. Can't wait to further update it, will only continue to get faster.
This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex. ~~This script does parse __.p4ignore__ ignore files, compiles the fields as regex, and scans every directory and file against the local and parent __.p4ignore__ files. This is my first time doing something like this, and I just realized this isn't actually correct; I need to update how things are ignored to follow the [spec](http://www.perforce.com/perforce/r12.1/manuals/cmdref/env.P4IGNORE.html), since it's not straight up regex.~~ I need to re-add this to the newer script.
**Files are currently permanently deleted, so use this at your own risk.** **Files are currently permanently deleted, so use this at your own risk.**

398
p4Helper.py Normal file
View File

@ -0,0 +1,398 @@
#!/usr/bin/python
# -*- coding: utf8 -*-
# author : Brian Ernst
# python_version : 2.7.6 and 3.4.0
# =================================
import datetime, inspect, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading
# trying ntpath, need to test on linux
import ntpath
try: input = raw_input
except: pass
#==============================================================
re_remove_comment = re.compile( "#.*$" )
def remove_comment( s ):
return re.sub( re_remove_comment, "", s )
def singular_pulural( val, singular, plural ):
return singular if val == 1 else plural
#==============================================================
def enum(*sequential, **named):
enums = dict(zip(sequential, range(len(sequential))), **named)
return type('Enum', (), enums)
MSG = enum('SHUTDOWN', 'PARSE_DIRECTORY', 'RUN_FUNCTION')
p4_ignore = ".p4ignore"
main_pid = os.getpid( )
#==============================================================
#if os.name == 'nt' or sys.platform == 'cygwin'
def basename( path ):
# TODO: import based on platform
# https://docs.python.org/2/library/os.path.html
# posixpath for UNIX-style paths
# ntpath for Windows paths
# macpath for old-style MacOS paths
# os2emxpath for OS/2 EMX paths
#return os.path.basename( path )
return ntpath.basename( path )
def normpath( path ):
return ntpath.normpath( path )
def join( patha, pathb ):
return ntpath.join( patha, pathb )
def splitdrive( path ):
return ntpath.splitdrive( path )
#==============================================================
def get_ignore_list( path, files_to_ignore ):
# have to split path and test top directory
dirs = path.split( os.sep )
ignore_list = [ ]
for i, val in enumerate( dirs ):
path_to_find = os.sep.join( dirs[ : i + 1] )
if path_to_find in files_to_ignore:
ignore_list.extend( files_to_ignore[ path_to_find ] )
return ignore_list
def match_in_ignore_list( path, ignore_list ):
for r in ignore_list:
if re.match( r, path ):
return True
return False
#==============================================================
def call_process( args ):
return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
def try_call_process( args, path=None ):
try:
subprocess.check_output( args.split( ), shell=False, cwd=path )
return 0
except subprocess.CalledProcessError:
return 1
use_bytearray_str_conversion = type( b"str" ) is not str
def get_str_from_process_stdout( line ):
if use_bytearray_str_conversion:
return ''.join( map( chr, line ) )
else:
return line
def parse_info_from_command( args, value, path = None ):
"""
:rtype : string
"""
proc = subprocess.Popen( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
if not line.startswith( value ):
continue
return line[ len( value ) : ].strip( )
return None
def get_p4_py_results( args, path = None ):
results = []
proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
try:
while True:
output = marshal.load( proc.stdout )
results.append( output )
except EOFError:
pass
finally:
proc.stdout.close()
return results
#==============================================================
def fail_if_no_p4():
if call_process( 'p4 -V' ) != 0:
print( 'Perforce Command-line Client(p4) is required for this script.' )
sys.exit( 1 )
# Keep these in mind if you have issues:
# https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime
# https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
def get_client_set( path ):
files = set( [ ] )
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
command = "p4 fstat ..."
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "... clientFile "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
if make_drive_upper:
drive, path = splitdrive( local_path )
local_path = ''.join( [ drive.upper( ), path ] )
files.add( local_path )
proc.wait( )
for line in proc.stderr:
if "no such file" in line:
continue
raise Exception(line)
return files
def get_client_root( ):
"""
:rtype : string
"""
command = "p4 info"
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "Client root: "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
return local_path
return None
class P4Workspace:
"""
Use this class when working in a workspace.
Makes sure the environmentals are setup correctly, and that you aren't working on a non-perforce directory accidentally;
otherwise you can delete files that shouldn't be deleted. Ex:
with P4Workspace( cwd ): #sets current workspace to cwd, or fails
# do stuff here
# on exit reverts to previous set workspace
"""
def __init__( self, directory):
self.directory = directory
def __enter__( self ):
# get user
#print("\nChecking p4 info...")
result = get_p4_py_results('info')
if len(result) == 0 or b'userName' not in result[0].keys():
print("Can't find perforce info, is it even setup?")
sys.exit(1)
username = get_str_from_process_stdout(result[0][b'userName'])
client_host = get_str_from_process_stdout(result[0][b'clientHost'])
#print("|Done.")
# see if current directory is set to current workspace, if not, set it to current workspace.
client_root = get_client_root()
ldirectory = self.directory.lower()
oldworkspace_name = None
if client_root is None or not ldirectory.startswith(client_root.lower()):
#print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...")
oldworkspace_name = parse_info_from_command('p4 info', 'Client name: ')
# get user workspaces
result = get_p4_py_results('workspaces -u ' + username)
workspaces = []
for r in result:
whost = get_str_from_process_stdout(r[b'Host'])
if whost is not None and len(whost) != 0 and client_host != whost:
continue
workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])}
workspaces.append(workspace)
del result
# check current directory against current workspace, see if it matches existing workspaces.
for w in workspaces:
wname = w['name']
wlower = w['root'].lower()
if ldirectory.startswith(wlower):
# set current directory, don't forget to revert it back to the existing one
#print("|Setting client view to: " + wname)
if try_call_process( 'p4 set P4CLIENT=' + wname ):
#print("|There was a problem trying to set the p4 client view (workspace).")
sys.exit(1)
break
else:
# TODO: look up workspace/users for this computer
print( "Couldn't find a workspace root that matches the current directory for the current user." )
sys.exit(1)
#print("|Done.")
self.oldworkspace_name = oldworkspace_name
return self
def __exit__( self, type, value, tb ):
# If we changed the current workspace, switch it back.
if self.oldworkspace_name is not None:
c.write("\nReverting back to original client view...")
# set workspace back to the original one
if try_call_process( 'p4 set P4CLIENT=' + self.oldworkspace_name ):
# error_count += 1 # have console log errors
# if not options.quiet:
print("There was a problem trying to restore the set p4 client view (workspace).")
sys.exit(1)
#else:
# if not options.quiet:
# c.write("|Reverted client view back to '" + self.oldworkspace_name + "'.")
#if not options.quiet:
# c.write("|Done.")
#==============================================================
class PTable( list ):
def __init__( self, *args ):
list.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
class PDict( dict ):
def __init__( self, *args ):
dict.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
#==============================================================
# TODO: Create a child thread for triggering autoflush events
# TODO: Hook console into stdout so it catches print
class Console( threading.Thread ):
MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' )
# auto_flush_time is time in milliseconds since last flush to trigger a flush when writing
def __init__( self, auto_flush_num = None, auto_flush_time = None ):
threading.Thread.__init__( self )
self.buffers = {}
self.buffer_write_times = {}
self.running = True
self.queue = multiprocessing.JoinableQueue( )
self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1
self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
self.shutting_down = False
def write( self, data, pid = None ):
self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) )
def writeflush( self, data, pid = None ):
pid = pid if pid is not None else os.getpid()
self.queue.put( ( Console.MSG.WRITE, pid, data ) )
self.queue.put( ( Console.MSG.FLUSH, pid ) )
def flush( self, pid = None ):
self.queue.put( ( Console.MSG.FLUSH, pid if pid is not None else os.getpid() ) )
def clear( self, pid = None ):
self.queue.put( ( Console.MSG.CLEAR, pid if pid is not None else os.getpid() ) )
def __enter__( self ):
self.start( )
return self
def __exit__( self, type, value, tb ):
self.queue.put( ( Console.MSG.SHUTDOWN, ) )
self.queue.join( )
def run( self ):
while True:
data = self.queue.get( )
event = data[0]
if event == Console.MSG.SHUTDOWN:
# flush remaining buffers before shutting down
for ( pid, buffer ) in self.buffers.items( ):
for line in buffer:
print( line )
self.buffers.clear( )
self.buffer_write_times.clear( )
self.queue.task_done( )
#print(self.queue.qsize())
#print(self.queue.empty())
break
elif event == Console.MSG.WRITE:
pid, s = data[ 1 : ]
if pid not in self.buffers:
self.buffers[ pid ] = []
if pid not in self.buffer_write_times:
self.buffer_write_times[ pid ] = datetime.datetime.now( )
self.buffers[ pid ].append( s )
if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num:
self.flush( pid )
elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time:
self.flush( pid )
# TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time
elif event == Console.MSG.FLUSH:
pid = data[ 1 ]
if pid in self.buffers:
for line in self.buffers[ pid ]:
print( line )
self.buffers.pop( pid, None )
self.buffer_write_times[ pid ] = datetime.datetime.now( )
elif event == Console.MSG.CLEAR:
pid = data[ 1 ]
if pid in self.buffers:
self.buffers.pop( pid, None )
self.queue.task_done( )
#==============================================================
# class Task( threading.Event ):
# def __init__( data, cmd = None ):
# threading.Event.__init__( self )
# self.cmd = cmd if cmd is None MSG.RUN_FUNCTION
# self.data = data
# def isDone( self ):
# return self.isSet()
# def join( self ):
# self.wait( )
#==============================================================
class Worker( threading.Thread ):
def __init__( self, console, queue, commands ):
threading.Thread.__init__( self )
self.queue = queue
self.commands = commands
def run( self ):
while True:
( cmd, data ) = self.queue.get( )
if not self.commands[cmd](data):
self.queue.task_done( )
break
self.queue.task_done( )

BIN
p4Helper.pyc Normal file

Binary file not shown.

View File

@ -11,317 +11,16 @@
# todo: buffer output, after exceeding a certain amount print to the output. # todo: buffer output, after exceeding a certain amount print to the output.
# todo: allow logging output besides console output, or redirection altogether # todo: allow logging output besides console output, or redirection altogether
import datetime, inspect, marshal, multiprocessing, optparse, os, re, stat, subprocess, sys, threading, time, traceback from p4Helper import *
# trying ntpath, need to test on linux import time, traceback
import ntpath
re_remove_comment = re.compile( "#.*$" ) #==============================================================
def remove_comment( s ):
return re.sub( re_remove_comment, "", s )
try: input = raw_input
except: pass
def enum(*sequential, **named):
enums = dict(zip(sequential, range(len(sequential))), **named)
return type('Enum', (), enums)
MSG = enum('SHUTDOWN', 'PARSE_DIRECTORY', 'RUN_FUNCTION')
p4_ignore = ".p4ignore"
main_pid = os.getpid( )
#if os.name == 'nt' or sys.platform == 'cygwin'
def basename( path ):
# TODO: import based on platform
# https://docs.python.org/2/library/os.path.html
# posixpath for UNIX-style paths
# ntpath for Windows paths
# macpath for old-style MacOS paths
# os2emxpath for OS/2 EMX paths
#return os.path.basename( path )
return ntpath.basename( path )
def normpath( path ):
return ntpath.normpath( path )
def join( patha, pathb ):
return ntpath.join( patha, pathb )
def splitdrive( path ):
return ntpath.splitdrive( path )
def get_ignore_list( path, files_to_ignore ):
# have to split path and test top directory
dirs = path.split( os.sep )
ignore_list = [ ]
for i, val in enumerate( dirs ):
path_to_find = os.sep.join( dirs[ : i + 1] )
if path_to_find in files_to_ignore:
ignore_list.extend( files_to_ignore[ path_to_find ] )
return ignore_list
def match_in_ignore_list( path, ignore_list ):
for r in ignore_list:
if re.match( r, path ):
return True
return False
def call_process( args ):
return subprocess.call( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
def try_call_process( args, path=None ):
try:
subprocess.check_output( args.split( ), shell=False, cwd=path )
return 0
except subprocess.CalledProcessError:
return 1
use_bytearray_str_conversion = type( b"str" ) is not str
def get_str_from_process_stdout( line ):
if use_bytearray_str_conversion:
return ''.join( map( chr, line ) )
else:
return line
def singular_pulural( val, singular, plural ):
return singular if val == 1 else plural
def parse_info_from_command( args, value, path = None ):
"""
:rtype : string
"""
proc = subprocess.Popen( args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
if not line.startswith( value ):
continue
return line[ len( value ) : ].strip( )
return None
def get_p4_py_results( args, path = None ):
results = []
proc = subprocess.Popen( [ 'p4', '-G' ] + args.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
try:
while True:
output = marshal.load( proc.stdout )
results.append( output )
except EOFError:
pass
finally:
proc.stdout.close()
return results
# Keep these in mind if you have issues:
# https://stackoverflow.com/questions/16557908/getting-output-of-a-process-at-runtime
# https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running
def get_client_set( path ):
files = set( [ ] )
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
command = "p4 fstat ..."
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "... clientFile "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
if make_drive_upper:
drive, path = splitdrive( local_path )
local_path = ''.join( [ drive.upper( ), path ] )
files.add( local_path )
proc.wait( )
for line in proc.stderr:
if "no such file" in line:
continue
raise Exception(line)
return files
def get_client_root( ):
"""
:rtype : string
"""
command = "p4 info"
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
for line in proc.stdout:
line = get_str_from_process_stdout( line )
clientFile_tag = "Client root: "
if not line.startswith( clientFile_tag ):
continue
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
return local_path
return None
class PTable( list ):
def __init__( self, *args ):
list.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
class PDict( dict ):
def __init__( self, *args ):
dict.__init__( self, args )
self.mutex = multiprocessing.Semaphore( )
# TODO: Create a child thread for triggering autoflush events
class Console( threading.Thread ):
MSG = enum('WRITE', 'FLUSH', 'SHUTDOWN', 'CLEAR' )
# auto_flush_time is time in milliseconds since last flush to trigger a flush when writing
def __init__( self, auto_flush_num = None, auto_flush_time = None ):
threading.Thread.__init__( self )
self.buffers = {}
self.buffer_write_times = {}
self.running = True
self.queue = multiprocessing.JoinableQueue( )
self.auto_flush_num = auto_flush_num if auto_flush_num is not None else -1
self.auto_flush_time = auto_flush_time * 1000 if auto_flush_time is not None else -1
self.shutting_down = False
def write( self, data, pid = None ):
self.queue.put( ( Console.MSG.WRITE, pid if pid is not None else os.getpid(), data ) )
def writeflush( self, data, pid = None ):
pid = pid if pid is not None else os.getpid()
self.queue.put( ( Console.MSG.WRITE, pid, data ) )
self.queue.put( ( Console.MSG.FLUSH, pid ) )
def flush( self, pid = None ):
self.queue.put( ( Console.MSG.FLUSH, pid if pid is not None else os.getpid() ) )
def clear( self, pid = None ):
self.queue.put( ( Console.MSG.CLEAR, pid if pid is not None else os.getpid() ) )
def __enter__( self ):
self.start( )
return self
def __exit__( self, type, value, tb ):
self.queue.put( ( Console.MSG.SHUTDOWN, ) )
self.queue.join( )
def run( self ):
while True:
data = self.queue.get( )
event = data[0]
if event == Console.MSG.SHUTDOWN:
# flush remaining buffers before shutting down
for ( pid, buffer ) in self.buffers.items( ):
for line in buffer:
print( line )
self.buffers.clear( )
self.buffer_write_times.clear( )
self.queue.task_done( )
#print(self.queue.qsize())
#print(self.queue.empty())
break
elif event == Console.MSG.WRITE:
pid, s = data[ 1 : ]
if pid not in self.buffers:
self.buffers[ pid ] = []
if pid not in self.buffer_write_times:
self.buffer_write_times[ pid ] = datetime.datetime.now( )
self.buffers[ pid ].append( s )
if self.auto_flush_num >= 0 and len( self.buffers[ pid ] ) >= self.auto_flush_num:
self.flush( pid )
elif self.auto_flush_time >= 0 and ( datetime.datetime.now( ) - self.buffer_write_times[ pid ] ).microseconds >= self.auto_flush_time:
self.flush( pid )
# TODO: if buffer is not empty and we don't auto flush on write, sleep until a time then auto flush according to auto_flush_time
elif event == Console.MSG.FLUSH:
pid = data[ 1 ]
if pid in self.buffers:
for line in self.buffers[ pid ]:
print( line )
self.buffers.pop( pid, None )
self.buffer_write_times[ pid ] = datetime.datetime.now( )
elif event == Console.MSG.CLEAR:
pid = data[ 1 ]
if pid in self.buffers:
self.buffers.pop( pid, None )
self.queue.task_done( )
# class Task( threading.Event ):
# def __init__( data, cmd = None ):
# threading.Event.__init__( self )
# self.cmd = cmd if cmd is None MSG.RUN_FUNCTION
# self.data = data
# def isDone( self ):
# return self.isSet()
# def join( self ):
# self.wait( )
class Worker( threading.Thread ):
def __init__( self, console, queue, files_to_ignore ):
threading.Thread.__init__( self )
self.console = console
self.queue = queue
self.files_to_ignore = files_to_ignore
def run( self ):
while True:
( cmd, data ) = self.queue.get( )
if cmd == MSG.SHUTDOWN:
self.console.flush( )
self.queue.task_done( )
break
if cmd == MSG.RUN_FUNCTION:
break
if cmd != MSG.PARSE_DIRECTORY or data is None:
self.console.flush( )
self.queue.task_done( )
continue
directory = data
# add threading stuffs
self.queue.task_done( )
def main( args ): def main( args ):
start = time.clock() start = time.clock()
# check requirements fail_if_no_p4()
if call_process( 'p4 -V' ) != 0:
print( 'Perforce Command-line Client(p4) is required for this script.' )
sys.exit( 1 )
#http://docs.python.org/library/optparse.html #http://docs.python.org/library/optparse.html
parser = optparse.OptionParser( ) parser = optparse.OptionParser( )
@ -336,184 +35,123 @@ def main( args ):
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) ) directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
# get user
print("\nChecking p4 info...")
result = get_p4_py_results('info')
if len(result) == 0 or b'userName' not in result[0].keys():
print("Can't find perforce info, is it even setup?")
sys.exit(1)
username = get_str_from_process_stdout(result[0][b'userName'])
client_host = get_str_from_process_stdout(result[0][b'clientHost'])
print("|Done.")
client_root = get_client_root()
ldirectory = directory.lower()
workspace_name = None
if client_root is None or not ldirectory.startswith(client_root.lower()):
print("\nCurrent directory not in client view, checking other workspaces for user '" + username + "' ...")
workspace_name = parse_info_from_command('p4 info', 'Client name: ')
# get user workspaces
result = get_p4_py_results('workspaces -u ' + username)
workspaces = []
for r in result:
whost = get_str_from_process_stdout(r[b'Host'])
if whost is not None and len(whost) != 0 and client_host != whost:
continue
workspace = {'root': get_str_from_process_stdout(r[b'Root']), 'name': get_str_from_process_stdout(r[b'client'])}
workspaces.append(workspace)
del result
# check current directory against current workspace, see if it matches existing workspaces.
for w in workspaces:
wname = w['name']
wlower = w['root'].lower()
if ldirectory.startswith(wlower):
# set current directory, don't forget to revert it back to the existing one
print("|Setting client view to: " + wname)
if try_call_process( 'p4 set P4CLIENT=' + wname ):
print("|There was a problem trying to set the p4 client view (workspace).")
sys.exit(1)
break
else:
print( "|Couldn't find a workspace root that matches the current directory for the current user." )
sys.exit(1)
print("|Done.")
# Files are added from .p4ignore
# Key is the file root, the value is the table of file regexes for that directory.
files_to_ignore = PDict()
processed_file_count = 0
processed_directory_count = 0
remove_file_count = 0
remove_dir_count = 0
warning_count = 0
error_count = 0
with Console( auto_flush_num=20, auto_flush_time=1000 ) as c: with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
if not options.quiet: with P4Workspace( directory ):
c.writeflush( "\nCaching files in depot, this may take a little while..." ) # Files are added from .p4ignore
# Key is the file root, the value is the table of file regexes for that directory.
files_to_ignore = PDict()
# TODO: push this off to a thread and walk the directory so we get a headstart. processed_file_count = 0
files_in_depot = get_client_set( directory ) processed_directory_count = 0
c.writeflush( "|Done." ) remove_file_count = 0
remove_dir_count = 0
# TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory? warning_count = 0
error_count = 0
# TODO: at this point join on both tasks to wait until they're done
# TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
# need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
# will at least need to redo how the ignore lists are handled for efficiencies sake.
if not options.quiet:
c.writeflush( "\nChecking " + directory)
for root, dirs, files in os.walk( directory ):
ignore_list = get_ignore_list( root, files_to_ignore )
if not options.quiet: if not options.quiet:
c.write( "|Checking " + os.path.relpath( root, directory ) ) c.writeflush( "\nCaching files in depot, this may take a little while..." )
for d in dirs: # TODO: push this off to a thread and walk the directory so we get a headstart.
processed_directory_count += 1 files_in_depot = get_client_set( directory )
path = join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ): c.writeflush( "|Done." )
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
for f in files: # TODO: push a os.walk request off to a thread to build a list of files in the directory; create batch based on directory?
processed_file_count += 1
path = normpath( join( root, f ) )
if path not in files_in_depot: # TODO: at this point join on both tasks to wait until they're done
if not options.quiet:
c.write( "| " + f + " is unversioned, removing it." ) # TODO: kick off file removal, make batches from the files for threads to work on since testing has to be done for each.
# need to figure out the best way to do this since the ignore list needs to be properly built for each directory;
# will at least need to redo how the ignore lists are handled for efficiencies sake.
if not options.quiet:
c.writeflush( "\nChecking " + directory)
for root, dirs, files in os.walk( directory ):
ignore_list = get_ignore_list( root, files_to_ignore )
if not options.quiet:
c.write( "|Checking " + os.path.relpath( root, directory ) )
for d in dirs:
processed_directory_count += 1
path = join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ):
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
for f in files:
processed_file_count += 1
path = normpath( join( root, f ) )
if path not in files_in_depot:
if not options.quiet:
c.write( "| " + f + " is unversioned, removing it." )
try:
os.chmod( path, stat.S_IWRITE )
os.remove( path )
remove_file_count += 1
except OSError as ex:
c.writeflush( "| " + type( ex ).__name__ )
c.writeflush( "| " + repr( ex ) )
c.writeflush( "| ^ERROR^" )
error_count += 1
if not options.quiet:
c.write( "|Done." )
if not options.quiet:
c.write( os.linesep + "Removing empty directories...")
# remove empty directories in reverse order
for root, dirs, files in os.walk( directory, topdown=False ):
ignore_list = get_ignore_list( root, files_to_ignore )
for d in dirs:
processed_directory_count += 1
path = os.path.join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ):
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
try: try:
os.chmod( path, stat.S_IWRITE ) os.rmdir(path)
os.remove( path ) remove_dir_count += 1
remove_file_count += 1 if not options.quiet:
except OSError as ex: c.write( "| " + rel_path + " was removed." )
c.writeflush( "| " + type( ex ).__name__ ) except OSError:
c.writeflush( "| " + repr( ex ) ) # Fails on non-empty directory
c.writeflush( "| ^ERROR^" ) pass
error_count += 1
if not options.quiet:
c.write( "|Done." )
if not options.quiet:
c.write( os.linesep + "Removing empty directories...")
# remove empty directories in reverse order
for root, dirs, files in os.walk( directory, topdown=False ):
ignore_list = get_ignore_list( root, files_to_ignore )
for d in dirs:
processed_directory_count += 1
path = os.path.join( root, d )
rel_path = os.path.relpath( path, directory )
if match_in_ignore_list( path, ignore_list ):
# add option of using send2trash
if not options.quiet:
c.write( "| ignoring " + rel_path )
dirs.remove( d )
try:
os.rmdir(path)
remove_dir_count += 1
if not options.quiet:
c.write( "| " + rel_path + " was removed." )
except OSError:
# Fails on non-empty directory
pass
if not options.quiet:
c.write( "|Done." )
# This needs to happen automatically even when an exception happens, when we leave scope.
if workspace_name is not None:
c.write("\nReverting back to original client view...")
# set workspace back to the original one
if try_call_process( 'p4 set P4CLIENT=' + workspace_name ):
error_count += 1
if not options.quiet:
c.write("|There was a problem trying to restore the set p4 client view (workspace).")
else:
if not options.quiet:
c.write("|Reverted client view back to '" + workspace_name + "'.")
if not options.quiet: if not options.quiet:
c.write("|Done.") c.write( "|Done." )
if not options.quiet: if not options.quiet:
output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " ) output = "\nChecked " + str( processed_file_count ) + singular_pulural( processed_file_count, " file, ", " files, " )
output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories") output += str( processed_directory_count ) + singular_pulural( processed_directory_count, " directory", " directories")
output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " ) output += "\nRemoved " + str( remove_file_count ) + singular_pulural( remove_file_count, " file, ", " files, " )
output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories") output += str( remove_dir_count ) + singular_pulural( remove_dir_count, " directory", " directories")
if warning_count > 0: if warning_count > 0:
output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" ) output += " w/ " + str( warning_count ) + singular_pulural( warning_count, " warning", " warnings" )
if error_count > 0: if error_count > 0:
output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" ) output += " w/ " + str( error_count ) + singular_pulural( error_count, " error", " errors" )
end = time.clock() end = time.clock()
delta = end - start delta = end - start
output += "\nFinished in " + str(delta) + "s" output += "\nFinished in " + str(delta) + "s"
c.write( output ) c.write( output )
if __name__ == "__main__": if __name__ == "__main__":
try: try:
main( sys.argv ) main( sys.argv )
except: except:
print( "Unexpected error!" ) print( "\nUnexpected error!" )
traceback.print_exc( file = sys.stdout ) traceback.print_exc( file = sys.stdout )

136
p4SyncMissingFiles.py Normal file
View File

@ -0,0 +1,136 @@
#!/usr/bin/python
# -*- coding: utf8 -*-
# author : Brian Ernst
# python_version : 2.7.6 and 3.4.0
# =================================
# TODO: setup batches before pushing to threads and use p4 --parallel
# http://www.perforce.com/perforce/r14.2/manuals/cmdref/p4_sync.html
from p4Helper import *
import time, traceback
#==============================================================
def main( args ):
start = time.clock()
fail_if_no_p4()
#http://docs.python.org/library/optparse.html
parser = optparse.OptionParser( )
parser.add_option( "-d", "--dir", dest="directory", help="Desired directory to crawl.", default=None )
parser.add_option( "-t", "--threads", dest="thread_count", help="Number of threads to crawl your drive and poll p4.", default=100 )
parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="This overrides verbose", default=False )
parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False )
parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False )
( options, args ) = parser.parse_args( args )
directory = normpath( options.directory if options.directory is not None else os.getcwd( ) )
with Console( auto_flush_num=20, auto_flush_time=1000 ) as c:
with P4Workspace( directory ):
if not options.quiet:
c.writeflush( "Preparing to sync missing files..." )
c.write( " Setting up threads..." )
# Setup threading
WRK = enum( 'SHUTDOWN', 'SYNC' )
def shutdown( data ):
return False
def sync( data ):
if data is not None and not os.path.exists( data ):
try_call_process( "p4 sync -f " + data )
if not options.quiet:
c.write( " Synced " + data )
return True
commands = {
WRK.SHUTDOWN : shutdown,
WRK.SYNC : sync
}
threads = [ ]
thread_count = options.thread_count if options.thread_count > 0 else multiprocessing.cpu_count( ) + threads
queue = multiprocessing.JoinableQueue( )
for i in range( thread_count ):
t = Worker( c, queue, commands )
threads.append( t )
t.start( )
make_drive_upper = True if os.name == 'nt' or sys.platform == 'cygwin' else False
command = "p4 fstat ..."
if not options.quiet:
c.writeflush( " Checking files in depot, this may take some time for large depots..." )
proc = subprocess.Popen( command.split( ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=directory )
clientFile_tag = "... clientFile "
headAction_tag = "... headAction "
# http://www.perforce.com/perforce/r12.1/manuals/cmdref/fstat.html
accepted_actions = [ 'add', 'edit', 'branch', 'move/add', 'move\\add', 'integrate', 'import', 'archive' ] #currently not checked
rejected_actions = [ 'delete', 'move/delete', 'move\\delete', 'purge' ]
client_file = None
for line in proc.stdout:
line = get_str_from_process_stdout( line )
if client_file and line.startswith( headAction_tag ):
action = normpath( line[ len( headAction_tag ) : ].strip( ) )
if not any(action == a for a in rejected_actions):
if options.verbose:
c.write( " Checking " + os.path.relpath( local_path, directory ) )
queue.put( ( WRK.SYNC, local_path ) )
if line.startswith( clientFile_tag ):
client_file = None
local_path = normpath( line[ len( clientFile_tag ) : ].strip( ) )
if make_drive_upper:
drive, path = splitdrive( local_path )
client_file = ''.join( [ drive.upper( ), path ] )
if len(line.rstrip()) == 0:
client_file = None
proc.wait( )
for line in proc.stderr:
if "no such file" in line:
continue
#raise Exception(line)
c.write(line)#log as error
if not options.quiet:
c.writeflush( " Pushed work, now waiting for threads..." )
for i in range( thread_count ):
queue.put( ( WRK.SHUTDOWN, None ) )
for t in threads:
t.join( )
if not options.quiet:
c.write( "Done." )
end = time.clock()
delta = end - start
output = "\nFinished in " + str(delta) + "s"
c.writeflush( output )
if __name__ == "__main__":
try:
main( sys.argv )
except:
print( "\nUnexpected error!" )
traceback.print_exc( file = sys.stdout )