Improved crawling speed, also cleaned up the output.
So the speed of the script is much faster than before, though it actually still has much room for improvement, it will just be more complicated. Calling 'p4 fstat' on the entire directory will give you everything you need up front, it's just they're in depot paths, which makes thing a little annoying to parse when you have workspace mappings that move things around so the local path may differ from the depot path, and it becomes harder to determine 100% that you're referring to the same file. And I don't want to have to call p4 on every file to be sure of that, what I'm doing now is the easiest safest way to be sure of that, as far as I know. Another way to speed this up is to add thread crawlers, I'm just not yet sure with HDDs and SSDs how many threads is a good idea to use.
This commit is contained in:
parent
09a4811be4
commit
266c5555ba
|
@ -8,9 +8,6 @@
|
||||||
import inspect, os, re, subprocess, sys, traceback
|
import inspect, os, re, subprocess, sys, traceback
|
||||||
|
|
||||||
|
|
||||||
modules_path = os.path.normpath( os.path.realpath( __file__ ) )
|
|
||||||
|
|
||||||
|
|
||||||
re_remove_comment = re.compile( "#.*$" )
|
re_remove_comment = re.compile( "#.*$" )
|
||||||
def remove_comment( s ):
|
def remove_comment( s ):
|
||||||
return re.sub( re_remove_comment, "", s )
|
return re.sub( re_remove_comment, "", s )
|
||||||
|
@ -55,12 +52,17 @@ def main( ):
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
root = "."
|
||||||
|
root_full_path = os.getcwd()
|
||||||
p4_ignore = ".p4ignore"
|
p4_ignore = ".p4ignore"
|
||||||
|
|
||||||
for root, dirs, files in os.walk( '.' ):
|
# make sure script doesn't delete itself
|
||||||
|
|
||||||
print ("\nChecking '" + root + "' ...")
|
files_to_ignore[ root ] = [ re.compile( os.path.join( re.escape( root + os.sep ), os.path.basename( __file__ ) ) ) ]
|
||||||
|
|
||||||
|
for root, dirs, files in os.walk( root ):
|
||||||
|
|
||||||
|
print ( os.linesep + "Checking '" + root + "' ...")
|
||||||
|
|
||||||
if p4_ignore in files:
|
if p4_ignore in files:
|
||||||
file_regexes = []
|
file_regexes = []
|
||||||
|
@ -74,24 +76,49 @@ def main( ):
|
||||||
if len( new_line ) > 0:
|
if len( new_line ) > 0:
|
||||||
file_regexes.append( re.compile( os.path.join( re.escape( root + os.sep ), new_line ) ) )
|
file_regexes.append( re.compile( os.path.join( re.escape( root + os.sep ), new_line ) ) )
|
||||||
|
|
||||||
print( " appending ignores from " + path )
|
print( "|appending ignores from " + path )
|
||||||
files_to_ignore[ root ] = file_regexes
|
files_to_ignore[ root ] = file_regexes
|
||||||
|
|
||||||
|
|
||||||
ignore_list = get_ignore_list( root )
|
ignore_list = get_ignore_list( root )
|
||||||
|
|
||||||
|
#command = "p4 have \"" + root + os.sep + "*\""
|
||||||
|
command = "p4 fstat *"
|
||||||
|
|
||||||
|
print("|" + command)
|
||||||
|
|
||||||
|
os.chdir( root )
|
||||||
|
proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
|
||||||
|
(out, err) = proc.communicate()
|
||||||
|
os.chdir( root_full_path )
|
||||||
|
|
||||||
|
# For ease we're doing a weird solution, rebuilding the file list. This is so we only need to parse unadded files.
|
||||||
|
files = []
|
||||||
|
|
||||||
|
for line in err.split( os.linesep ):
|
||||||
|
if len(line) == 0:
|
||||||
|
continue
|
||||||
|
# # dirty hack that grabs the filename from the ends of the printed out (not err) "depo_path - local_path"
|
||||||
|
# # I could use regex to verify the expected string, but that will just slow us down.
|
||||||
|
# basename = os.path.basename( line )
|
||||||
|
i = line.rfind( ' - ')
|
||||||
|
if i >= 0:
|
||||||
|
basename = line[ : i ]
|
||||||
|
path = os.path.join( root, basename )
|
||||||
|
if not os.path.isdir( path ):
|
||||||
|
for file in files:
|
||||||
|
if file == basename:
|
||||||
|
files.append( file )
|
||||||
|
break
|
||||||
|
|
||||||
for file in files:
|
for file in files:
|
||||||
path = os.path.join( root, file )
|
path = os.path.join( root, file )
|
||||||
|
|
||||||
if match_in_ignore_list( path, ignore_list ):
|
if match_in_ignore_list( path, ignore_list ):
|
||||||
print( " ignoring " + path )
|
print( "| ignoring " + path )
|
||||||
continue
|
continue
|
||||||
|
|
||||||
proc = subprocess.Popen( "p4 fstat \"" + path + "\"", stdout=subprocess.PIPE, stderr=subprocess.PIPE )
|
print( "| " + file + " is unversioned, removing it." )
|
||||||
(out, err) = proc.communicate()
|
|
||||||
|
|
||||||
if len( err ) > 0:
|
|
||||||
print( " " + file + " is unversioned, removing it." )
|
|
||||||
os.remove( path )
|
os.remove( path )
|
||||||
|
|
||||||
dirs_copy = dirs
|
dirs_copy = dirs
|
||||||
|
@ -99,18 +126,22 @@ def main( ):
|
||||||
path = os.path.join( root, d )
|
path = os.path.join( root, d )
|
||||||
if match_in_ignore_list( path, ignore_list ):
|
if match_in_ignore_list( path, ignore_list ):
|
||||||
# add option of using send2trash
|
# add option of using send2trash
|
||||||
print( " ignoring " + d )
|
print( "| ignoring " + d )
|
||||||
dirs.remove( d )
|
dirs.remove( d )
|
||||||
|
|
||||||
|
print( "|Done." )
|
||||||
|
|
||||||
|
print( os.linesep + "Removing empty directories...")
|
||||||
# remove empty directories
|
# remove empty directories
|
||||||
for root, dirs, files in os.walk( '.', topdown=False ):
|
for root, dirs, files in os.walk( '.', topdown=False ):
|
||||||
for d in dirs:
|
for d in dirs:
|
||||||
try:
|
try:
|
||||||
os.rmdir(d)
|
os.rmdir(d)
|
||||||
print( " " + d + " was removed." )
|
print( "|" + d + " was removed." )
|
||||||
except OSError:
|
except OSError:
|
||||||
# Fails on non-empty directory
|
# Fails on non-empty directory
|
||||||
pass
|
pass
|
||||||
|
print( "|Done." )
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
try:
|
try:
|
||||||
|
|
Loading…
Reference in New Issue