Krdytkyu

I'm trying to compare some entries in a table to some entries built from some other tables in arcgis using python. When I ran similar code to this, the script took no longer than 15 minutes. I moved some things around in the code because I had to in order to write the type of comparison I wanted. As a result a certain list comprehension went from around 1.5k lists to 90k lists - I'm using a list of lists because I didn't know I had access to pandas and at this point I don't have time to learn pandas to do this. The script now takes over 3 hours to run and almost all of it is on this one list comprehension. In fact I haven't actually been able to get the script to move past the list comprehension since I made these changes.

Why is this taking so damn long and how do I fix this? Here is the relevant part of the "script":

import arcpy

import os

import csv

import time

import shutil





def compareLists(listOne, listTwo,):



    notInList = [item for item in listOne if item not in listTwo]

    print "Lists compared wp"

    return notInList



def compareDicts(dictOne, dictTwo):

    #should build a dictionary, no?

    #For some reason, the results for this are different when

    #dictOne is swapped with dictTwo. Why?

    dictCompare = {}

    for item in dictOne:

        areaMoreThan1Percent = ""

        lengthMoreThan1Percent = ""

        if item in dictTwo:

            if not (dictOne[item][0] < (dictTwo[item][0] * 1.01) and

            (dictOne[item][0] > 0.99)):

                areaMoreThan1Percent = "Check Area"

            if not (dictOne[item][1] < (dictTwo[item][1] * 1.01) and

            (dictOne[item][1] > 0.99)):

                lengthMoreThan1Percent = "Check Length"

            if areaMoreThan1Percent or lengthMoreThan1Percent:

                dictCompare.update({item: [areaMoreThan1Percent,

                                    lengthMoreThan1Percent] +

                                   dictOne[item]})

    print "Done with what's in {} and not {}".format(compareNameOne,

                                                    compareNameTwo)

    return dictCompare



#Making a FC to cut things down to the relevant records

Directory = r"U:Temp Connections"



# Calculate date, time, and FGDB name

date = time.strftime('%Y %m %d  %H %M %S')

GDB_Name = date + '_NewFootprints.gdb'



# Create a new FileGDB

arcpy.CreateFileGDB_management(Directory, GDB_Name)



# Set to path of created FileGDB

GDB = os.path.join(Directory, GDB_Name)



connection_name = "Real enterprise path"

database_platform = "SQL_SERVER"

instance = "realInstance"

authentication = "correctAuthForm"

username = "RightUserName"

password = "CorrectPassword"

savePW = "SAVE_USERNAME"

database = "SolidDatabase"



# Look for folder to put Prod Connection into

if not os.path.isdir(Directory):

    os.path.makedirs(Directory)



# Look for Prod connection and create if absent

if not os.path.isfile(os.path.join(Directory, connection_name)):

    print ("Making connection file")

    arcpy.CreateDatabaseConnection_management(Directory,

                                          connection_name,

                                          database_platform,

                                          instance,

                                          authentication,

                                          username,

                                          password,

                                          savePW,

                                          database)



easyFDS = r"realFeatureDataset"

FeatureClass = r"realFeatureClass"

FullPathFC = os.path.join(Directory, connection_name, easyFDS, FeatureClass)

sr = arcpy.Describe(FullPathFC).spatialReference



# Create new FDS for fc

arcpy.CreateFeatureDataset_management(GDB,

                                      "someStuff",

                                      sr)



# Get file path of FDS in created FileGDB

newFDS = os.path.join(GDB, "someStuff")



print ("New File GDB and feature dataset have been created")



difficultFC = os.path.join(r"first half of the connection path",

                      r"second half of the connection path")

easyFC = os.path.join(r"first half of the connection path",

                             r"Second half of the other connection path"

                             )

anotherFCForTheHardOne = r"rightConnectionPathForThis"





inFeaturesList = [hardFC, anotherFCForTheHardOne]

outFeatureClass = os.path.join(newFDS,"The Right Stuff")



arcpy.Intersect_analysis(inFeaturesList, outFeatureClass)

print "Intersect ran"



#list and dict comprehensions

easyFC = os.path.join(r"first half of the connection path",

                             r"Second half of the other connection path"

                             )

easyFields = ["common","OBJECTID","SHAPE.STArea()","SHAPE.STLength()"]

#Dunno if we're using OWNER_NUMBER

easyFieldsDict = ["common","OBJECTID","SHAPE.STArea()","SHAPE.STLength()",

                         "Field0", "Field1", "Field2", "Field3",

                         "Field3", "Field4","Field5",

                         "Field6", "Field7", "Field8", "Field9"]

easyList = [[str(row[0]),str(row[4]), str(row[5]), str(row[6]),

                   str(row[7]), str(row[8]), str(row[9]),str(row[10]),

                   str(row[11]), str(row[12]),str(row[13])]

                  for row in arcpy.da.SearchCursor(easyFC,

                                                   easyFieldsDict)]

easyDict = {str(row[0]): [float(row[2]), float(row[3])] for row in

                  arcpy.da.SearchCursor(easyFC, easyDict)}

print "Done with easy List"



hardFC = outFeatureClass

hardFields = ["common","OBJECTID","Shape_Area","Shape_Length"]

hardFieldsDict = ["common","OBJECTID","Shape_Area","Shape_Length"]

hardList = [str(row[0]) for row in arcpy.da.SearchCursor(hardFC, hardFields)]

hardDict = {str(row[0]): [float(row[2]) ,float(row[3])] for row in

            arcpy.da.SearchCursor(hardFC, hardFieldsDict)}

moreData = r"Correct connection path"

moreDataFields = ["common", "OBJECTID", "Field0", "Field1", "Field2",

                  "Field3", "Field4", "Field5", "Field6", "Field7",

                  "Field8", "Field9"]



#This takes forever.

#This one bit seems to be taking forever. What did I do!?



hardListComp = [[str(row[0]), str(row[1]), str(row[2]), str(row[11]),str(row[3]),

             str(row[4]), str(row[5]), str(row[6]), str(row[7]), str(row[8]),

             str(row[9]), str(row[10])] for item in hardList for row in

            arcpy.da.SearchCursor(moreData, moreDataFields,

                                  "common = '{}'".format(item))]

print "hardListComp made"

Is this just too big for a list? As always any comments on the poor quality of my code are welcome.

hardListComp finally finished running. It took about 3.5 hours.