from datetime import datetime, timedelta from django.utils import timezone from ideatree.itreeFirebase import getFirebaseInstance from django.core.management.base import BaseCommand, CommandError from django.conf import settings from django.db import transaction from ideatree.models import Node, Edge, NodeComment, Vote, Mapmember, Map_desc, Friend, MapPathEndpoints from django.db import IntegrityError from django.db.models import ProtectedError from firebase_admin import credentials, db, auth from ideatree.utils import myutils #import pdb class Command(BaseCommand): help = "Cleaning out db records marked as 'Delete', or Firebase db records older than a given time." def handle(self, *args, **options): try: # FIXME remove unused session keys from the db with clearsessions() # Django has Python DateTime db fields, but firebase data has time in milliseconds. Both must be in UTC. #offsetSecs = 0 # for testing offsetSecs = 0 - settings.PROVISIONAL_SECONDS_TO_STALE self.delBeforeTimeInMilliseconds = myutils.millisNow(offsetSecs) # IMPORTANT: set client-side provisional expiration to PROVISIONAL_SECONDS_TO_STALE time. # One second is deducted here because the firebase end_at() filter below is inclusive. # Cutoff is in the past (whatever 'stale' is set to): # NOTE: Be sure this time zone matches what the database is set to, and that the database matches what the OS is set to. self.cutoffTime = (datetime.now(timezone.utc) - timedelta(seconds=(settings.PROVISIONAL_SECONDS_TO_STALE -1))).strftime('%Y-%m-%d %H:%M:%S%z') self.stdout.write("Cleaning out data older than " + str(self.cutoffTime) + " UTC") ready4FirebaseClean = False with transaction.atomic(): try: # Doing nodes first will automatically cascade delete of a lot of the others, including some objects not explicitly # handled here. See models.py for cascades. self.cleanOutFriends() self.cleanOutNodes() self.cleanOutEdges() self.cleanOutNodeComments() self.cleanOutNodeVotes() self.cleanOutMapMembers() self.checkDanglingMapPathEndpoints() ready4FirebaseClean = True except ProtectedError as err: self.stdout.write("ProtectedError: cannot delete:") for obj in err.protected_objects: self.stdout.write(str(obj._meta) + " id:" + str(obj.id)) except IntegrityError as err: self.stdout.write("IntegrityError: " + str(err)) if ready4FirebaseClean: self.cleanOutFirebase() # MUCH SLOWER than the above, so run outside the transaction. # Now we can remove maps marked for deletion, since we don't need them to search on Firebase. self.cleanOutMaps() except CommandError as err: # FIXME: what kind of exceptions does the Firebase admin SDK raise? self.stdout.write(str(err)) except Exception as err: # FIXME: what kind of exceptions does the Firebase admin SDK raise? self.stdout.write(str(err)) def checkDanglingMapPathEndpoints(self): # Check that the MapPathEndpoint record was automatically deleted when the endpoint fields were both set to None when those nodes were deleted. allmaps = Map_desc.objects.filter(status=settings.MAP_ACTIVE) danglingPathList = [] for amap in allmaps: # Look for only path objects whose endpoints were both deleted (there should be none) danglingPaths = MapPathEndpoints.objects.filter(ofmap=amap, pathBeginNode=None, pathEndNode=None).count() if danglingPaths > 0: danglingPathList.append(danglingPath.id) if len(danglingPathList): raise Exception("ERROR: Dangling MapPathEndpoint object(s) found. Ids: " + str(danglingPathList)) self.stdout.write("No dangling MapPathEndpoints found.") def cleanOutFriends(self): try: Friend.objects.filter(status=settings.FRIEND_DELETED).delete() self.stdout.write("Deleted Friends cleaned out.") except: raise def cleanOutNodeComments(self): try: NodeComment.objects.filter(status=settings.NODE_COMMENT_DELETED, date__lt = self.cutoffTime).delete() self.stdout.write("Deleted node comments cleaned out.") except: raise def cleanOutNodeVotes(self): try: Vote.objects.filter(status=settings.VOTABLE_DELETED, date__lt = self.cutoffTime).delete() self.stdout.write("Deleted node votes cleaned out.") except: raise def cleanOutNodes(self): try: Node.objects.filter(status=settings.NODE_DELETED, created_date__lt = self.cutoffTime).delete() Node.objects.filter(nodetype=settings.PROVISIONAL, created_date__lt = self.cutoffTime).delete() self.stdout.write("Deleted or stale nodes cleaned out.") except: raise def cleanOutEdges(self): try: Edge.objects.filter(status=settings.EDGE_DELETED).delete() Edge.objects.filter(status=settings.EDGE_PROVISIONAL, creation_date__lt = self.cutoffTime).delete() self.stdout.write("Deleted or stale edges cleaned out.") except: raise def cleanOutMapMembers(self): try: Mapmember.objects.filter(status=settings.MAPMEMBERSHIP_DELETED).delete() self.stdout.write("Deleted map sharers cleaned out.") except: raise def cleanOutMaps(self): try: Map_desc.objects.filter(status=settings.MAP_DELETED).delete() self.stdout.write("Deleted maps cleaned out.") except: raise def cleanOutFirebase(self): # Delete stale data on Firebase. try: self.stdout.write("Deleting stale Firebase data...") fb = getFirebaseInstance() maps = Map_desc.objects.all() for amap in maps: dbBranch = str(amap.id) print("Checking: " + dbBranch) ref = db.reference("changeMessages/" + dbBranch) # NOTE: assumes Firebase data is ordered by time in epoch milliseconds in UTC timezone. # NOTE: requires an .indexOn:'time' rule on Firebase snapshot = ref.order_by_child('time').end_at(self.delBeforeTimeInMilliseconds).get() numitems = len(snapshot) self.stdout.write(str(numitems) + " objects FOUND. Map " + dbBranch) if numitems: for item in snapshot: ref.child(item).delete() self.stdout.write(" " + str(numitems) + " objects DELETED. Map " + dbBranch) # chat messages for amap in maps: dbBranch = str(amap.id) ref = db.reference("chat/" + dbBranch) # NOTE: assumes Firebase data is ordered by time in epoch milliseconds in UTC timezone. # NOTE: requires an .indexOn:'time' rule on Firebase snapshot = ref.order_by_child('time').end_at(self.delBeforeTimeInMilliseconds).get() numitems = len(snapshot) self.stdout.write(str(numitems) + " chat messages FOUND. Map " + dbBranch) if numitems: for item in snapshot: ref.child(item).delete() self.stdout.write(" " + str(numitems) + " chat messages DELETED. Map " + dbBranch) except ValueError as err: raise Exception("Firebase ValueError.") except Exception as err: raise