Add memoisation

download_data.py

@@ -24,13 +24,18 @@ base_types = {

                                
                                
                                
                            
                                
                                    
                                        
                                            24  
                                                "waste": set(),
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            25  
                                            }
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            26  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        27 +
                                        memoisation = {}
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        28 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            29  
                                            def get_objects(object_info):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        30 +
                                            if object_info["id"] in memoisation:
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        31 +
                                                return memoisation[object_info["id"]]
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            32  
                                                objects = set()
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            33  
                                                for object in object_info["children"]:
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            34  
                                                    #print(f"Scanning {object}")
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            35  
                                                    info = httpx.get("https://datasets.roundabout-host.com/api/object/" + object).json()
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            36  
                                                    objects |= get_objects(info)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            37  
                                                objects.add(object_info["id"])
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        38 +
                                            memoisation[object_info["id"]] = objects
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            39  
                                                return objects
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            40  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            41  
                                            for object_name in base_types: