import glob import os import shutil #FIXME import stat import json import tempfile import time # FIXME: needed? import re import stripe from django.views.decorators.csrf import csrf_exempt # FIXME: remove as many "from" imports as possible to limit things in global namespace import shlex from subprocess import run, check_output, CalledProcessError from string import whitespace from django.contrib.auth import logout from django.views.decorators.csrf import csrf_protect from ideatree.itreeFirebase import getFirebaseInstance, firebaseClientToken, push_value from ideatree.utils import myutils from django.contrib.auth.signals import user_logged_out from django.dispatch import receiver from copy import deepcopy from django.core import serializers from datetime import datetime, timedelta # FIXME: needed? # For now, client receives UTC times in templates. If this is changed, see the excellent https://docs.djangoproject.com/en/1.11/topics/i18n/timezones/#time-zones from django.utils import timezone # FIXME what defs are actually used? from dateutil.relativedelta import relativedelta from django.db import IntegrityError, transaction from django.core.mail import BadHeaderError from smtplib import SMTPException from django.conf import settings from django.http import HttpResponse, HttpResponseRedirect import http from django.shortcuts import get_object_or_404, render # FIXME from django.urls import reverse # FIXME: needed? from django.forms.utils import ErrorList from django.contrib.auth.models import User from django.core.mail import send_mail from .models import Node, Edge, Friend, Map_desc, Mapmember, Map_access_changed, UserProfile, WhosViewingMap, NodeComment, Vote, ClientPermission, WhosLoggedIn, ContactUs, MapPathEndpoints, Colors from .forms import ContactUsForm, OpenMapForm, NewNodeForm, NewTunnelNodeForm, NodeCommentForm, NewMapForm, DeleteMapForm, FriendTable, UserSelectTable, NotifyUsersForm, SearchAllUsersForm, MapSettingsForm, AlterEdgeForm, AlterMapForm, FriendDefineForm, MyAccountForm, VoteForm, FileUploadForm, ValidateNodeForm, ValidateEdgeForm, ConfirmCancelAccountForm, ValidateMapForm from django.contrib.auth.decorators import login_required from allauth.account.decorators import verified_email_required from django.views.decorators.csrf import ensure_csrf_cookie from django.contrib.admin.views.decorators import staff_member_required from django.forms import modelformset_factory # FIXME from . import tooltips from django.core.exceptions import PermissionDenied, ObjectDoesNotExist, FieldDoesNotExist, FieldError, ValidationError, SuspiciousOperation from collections import defaultdict # FIXME: defaultdict needed? from django.db.models import F from django_tables2 import RequestConfig from django.db.models import Q from django.http import JsonResponse import random import sys from django.contrib.auth.decorators import user_passes_test import pdb from pprint import pprint # FIXME: remove all style tags from templates # Security check: # FIXME: review and implement https://docs.djangoproject.com/en/1.11/ref/csrf/ # NOTE: tests.py runs this as a setup (using the test database) to everything else. So permissions during testing may differ from what you # see querying the real database directly if the real db hasn't been initialized by running this. @login_required() @staff_member_required def initPerms(request): from ideatree.utils import initPermissionsTable try: initPermissionsTable.initPermsTable(request) return HttpResponse("Success", status=200) except Exception as err: return HttpResponse(str(err), status=406) # NOTE: This is run ONE time on install of an empty database to load up the Colors table with values. @login_required() @staff_member_required def initColors(request): try: from ideatree.utils import importColorTable importColorTable.importColors() return HttpResponse("Success", status=200) except Exception as err: return HttpResponse(str(err), status=406) # FIXME: move these to a signals module and call from ready(). See https://docs.djangoproject.com/en/1.11/topics/signals/ # and https://docs.djangoproject.com/en/dev/ref/contrib/auth/#module-django.contrib.auth.signals # DEBUGGING WITH WSGI and APACHE # WSGI recommends no reading/writing to stdout to preserve portability (overrides print). # See https://modwsgi.readthedocs.io/en/develop/user-guides/debugging-techniques.html # DO THIS: # print('something', file=sys.stderr) def not_guest_user(user): # NOTE: REQUIRES that guest username be blacklisted for django-allauth in settings.py not_guest = not(user.username == settings.GUEST_USERNAME) if user: return (not_guest) else: return(False) # FIXME delete def showOutline(request): pass return True @login_required() def incrementViewers(request): try: # FIXME: form really the best way to validate? userId = int(request.session["_auth_user_id"]) mapId = int(json.loads(request.body.decode())["mapId"]) WhosLoggedIn.objects.filter(user_id=userId).delete() # NOTE: limitation: can only be logged in to one map at a time. WhosLoggedIn.objects.create(user_id=userId, currentmap_id=mapId) return HttpResponse("Success", status=200) except: raise @receiver(user_logged_out) def decrementViewers(sender, **kwargs): try: user = kwargs["user"] WhosLoggedIn.objects.filter(user=user).delete() return HttpResponse("Success", status=200) except: raise # SECURITY NOTE: Do NOT use class-based views. Here's why: # https://groups.google.com/g/django-developers/c/HUZySAw43uE/m/RD4ifBLPBgAJ and header "Discussion" at # https://spookylukey.github.io/django-views-the-right-way/anything.html def index(request): try: randomNum = random.randrange(1000000) enableGoogleAnalytics = False if not settings.DEBUG else False context = {'enableGoogleAnalytics':enableGoogleAnalytics, 'whatnodeiscalled':settings.WHAT_A_NODE_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'randomNum':randomNum} return render(request,'ideatree/networkGraphLanding.html', context) except Exception as err: return HttpResponse(str(err), status=406) @ensure_csrf_cookie @login_required() # FIXME: why is this here rather than in itreeFireebase.py like the rest? def firebaseClientCredentials(request): try: # NOTE: !!!!!!!!!!!!!! critical code for security. Get it Right.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # NOTE: because of security sensitivity, be sure these conditionals only respond to True/False, NOT truthy/falsy or 'None' if settings.USE_FIREBASE is True: if settings.USE_DEVEL_FIREBASE_SITE is True: print("Sending DEVELOPMENT Firebase credentials to client.", file=sys.stderr) fp = open(settings.FIREBASE_DEVELOPMENT_CLIENT_CREDENTIALS_PATH,'r') elif settings.USE_DEVEL_FIREBASE_SITE is not True: print("Sending PRODUCTION Firebase credentials to client.", file=sys.stderr) fp = open(settings.FIREBASE_PRODUCTION_CLIENT_CREDENTIALS_PATH,'r') content = fp.read() fp.close() return HttpResponse(content, content_type="text/javascript") except FileNotFoundError as err: raise Exception("FileNotFoundError:" + str(msg)) except Exception as err: return HttpResponse(str(err), status=406) def features(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'whatanodeiscalled':settings.WHAT_A_NODE_IS_CALLED } return render(request,'ideatree/features.html', context) except Exception as err: return HttpResponse(str(err), status=406) def shared_or_private(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'whatanodeiscalled':settings.WHAT_A_NODE_IS_CALLED } return render(request,'ideatree/shared_or_private.html', context) except Exception as err: return HttpResponse(str(err), status=406) def advanced_but_simple(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'whatanodeiscalled':settings.WHAT_A_NODE_IS_CALLED, 'what_an_edge_is_called':settings.WHAT_AN_EDGE_IS_CALLED } return render(request,'ideatree/advanced_but_simple.html', context) except Exception as err: return HttpResponse(str(err), status=406) def link_or_analyze(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } return render(request,'ideatree/link_or_analyze.html', context) except Exception as err: return HttpResponse(str(err), status=406) def videos(request): try: return render(request,'ideatree/videos.html') except Exception as err: return HttpResponse(str(err), status=406) def video_how_to_use(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } return render(request,'ideatree/video_how_to_use.html', context) except Exception as err: return HttpResponse(str(err), status=406) def video_for_project_managers(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } return render(request,'ideatree/video_for_project_managers.html', context) except Exception as err: return HttpResponse(str(err), status=406) # FIXME: this video shows old method of inviting #def video_inviting_users(request): #return render(request,'ideatree/video_inviting_users.html') def research(request): try: return render(request,'ideatree/research.html') except Exception as err: return HttpResponse(str(err), status=406) def press(request): try: return render(request,'ideatree/presskit.html') except Exception as err: return HttpResponse(str(err), status=406) @csrf_protect def contactus(request): try: userId = request.session["_auth_user_id"] if request.session.keys() else None loggedIn = userId is not None context = {'loggedIn':loggedIn, 'ideatrees_email': settings.EMAIL_OF_IDEATREE} if request.method == 'GET': """ Dead code since removing the template form due to spam. form = ContactUsForm(label_suffix='') context["form"] = form """ return render(request,'ideatree/contactus.html', context) """ if request.method == 'POST': form = ContactUsForm(request.POST, label_suffix='') context["form"] = form if not form.is_valid(): return render(request,'ideatree/contactus.html', context ) else: email = request.POST['email'] userMsg = request.POST['message'] if userId: ContactUs.objects.create(user_id=userId, email=email, message=userMsg) fromAddr = None subject="IDEATREE SUPPORT INQUIRY" msg="Message from: " + email + " :\n\n" + userMsg try: send_mail(subject, msg, fromAddr, [settings.EMAIL_OF_IDEATREE], fail_silently=False,) return render(request,'ideatree/contactusThankyou.html', context ) except Exception as err: context["error"] = str(err) return render(request,'ideatree/contactusThankyou.html', context ) """ except Exception as err: return HttpResponse(str(err), status=406) def terms_of_service(request): try: return render(request,'ideatree/terms-of-service.html') except Exception as err: return HttpResponse(str(err), status=406) def privacystatement(request): try: context={ 'paymentprocessor': 'Stripe', 'paymentprocessor_privacy_statement_url':"https://stripe.com/gb/privacy", 'real_time_db':'Google Firebase', 'real_time_db_privacy_statment':'https://firebase.google.com/support/privacy' } return render(request,'ideatree/privacystatement.html', context) except Exception as err: return HttpResponse(str(err), status=406) def idea_selection_paper(request): # FIXME: won't render a pdf try: return render(request,'ideatree/idea_selection_2.pdf') except Exception as err: return HttpResponse(str(err), status=406) """ VERY IMPORTANT: logout from any javascript login that was done in the homepage or another browser window in order to display a map by API. If you don't do this and they sign up, the custom value sent to PayPal will be that of the owner of the API-displayed map, NOT of this new signup. And the API owner's database entry will receive the activation from PayPal, NOT the new signup person. unset($_SESSION["userID"]); unset($_SESSION["username"]); unset($_SESSION["mapId"]); // FIXME: test this. """ def presignup(request): try: special_msg = "Included for early adopters" extra_fee = "By request. Additional fee applies." free_trial_days = settings.FREE_ACCT_TRIALPERIOD_DAYS + settings.FREE_ACCT_GRACEPERIOD_DAYS; features = [] features.append( ("Real-time collaboration (not just sharing)",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Collapsible branches",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("View graph as text outline",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Create tunnels between related graphs",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Link nodes to websites",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Shortest path analysis",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Free publishing to public website",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Excel import/export",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Cloud-based, no installation required",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Automatic free updates",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Voting",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Chat box",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Email notifications to co-editors",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]]) ) features.append( ("Comments on "+settings.WHAT_A_NODE_IS_CALLED+"s:",[[settings.FREE_ACCT,True], [settings.PREMIUM_ACCT,True]])) features.append( ("Number of "+settings.WHAT_A_GRAPH_IS_CALLED+"s: ",[[settings.FREE_ACCT,settings.FREE_USER_NUM_MAPS_ALLOWED], [settings.PREMIUM_ACCT,str(settings.PREMIUM_USER_NUM_MAPS_ALLOWED) + ' owned, plus unlimited shared']]) ) features.append( ("Number of "+settings.WHAT_A_NODE_IS_CALLED+ "s per "+settings.WHAT_A_GRAPH_IS_CALLED+": ",[[settings.FREE_ACCT,settings.MAX_FREE_ACCOUNT_NODES_PER_MAP], [settings.PREMIUM_ACCT,str(settings.MAX_PREMIUM_ACCOUNT_NODES_PER_MAP) ]]) ) features.append( ("Additional analysis functions",[[settings.FREE_ACCT, False], [settings.PREMIUM_ACCT,extra_fee]]) ) features.append( ("Additional import/export formats",[[settings.FREE_ACCT, False], [settings.PREMIUM_ACCT,extra_fee]]) ) features.append( ("Support",[[settings.FREE_ACCT, "How-to newsletter, contact us form"], [settings.PREMIUM_ACCT, "How-to newsletter, contact us form, plus initial half-hour one-on-one consultation" ]]) ) features.append( ("Browsers",[[settings.FREE_ACCT, "Latest versions of Chrome, Brave, Firefox, or Edge (not Safari)"], [settings.PREMIUM_ACCT, "Same" ]]) ) context = { 'features':features, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'premium_account_price': settings.PREMIUM_ACCT_PRICE, 'free_trial_days' : free_trial_days } return render(request,'ideatree/presignup.html', context) except Exception as err: return HttpResponse(str(err), status=406) # --------------------------------- PAYMENT PROCESSING ------------------------------------------------ @login_required() @user_passes_test(not_guest_user) def cancelAccount(request): # NOTE: the corresponding Stripe Customer account was created by signal in models.py user = User.objects.get(pk=int(request.session["_auth_user_id"])) # ignore what may be sent and always use the session context = {'username':user.username, 'paymentprocessor':settings.PAYMENT_PROCESSOR_NAME, 'inactiveUserDeletionDelay':settings.INACTIVE_USER_DELETION_DELAY_DAYS, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'email':user.email, 'first_name':user.first_name, 'last_name':user.last_name} try: if not request.POST.get("submitted"): # We can't check for GET/POST because with Ajax everything is a POST. form = ConfirmCancelAccountForm() context.update({'form':form}) return render(request, 'ideatree/cancelAccount.html', context) else: form = ConfirmCancelAccountForm(request.POST) context.update({'form':form}) if not form.is_valid(): return render(request, 'ideatree/cancelAccount.html', {'form':form, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED }) else: if not form.cleaned_data["iHaveReadAndUnderstood"] == True: form.add_error("iHaveReadAndUnderstood", "This is required.") if not form.cleaned_data["confirmcancel"] == settings.CONFIRM_DELETE_PHRASE: form.add_error("confirmcancel", "The typed phrase did not match the prompt.") if form.errors: return render(request, 'ideatree/cancelAccount.html', context) else: user_profile = UserProfile.objects.get(user=user) globalDelete = form.cleaned_data["deleteDataInNonOwnedMaps"] if globalDelete: user_profile.deleteglobally=True user_profile.save() stripecustomerid = None paidAccount = user_profile.accounttype in [settings.REGULAR_ACCT, settings.PREMIUM_ACCT] if paidAccount: stripecustomerid = user_profile.stripecustomerid if not stripecustomerid: raise Exception("No Stripe customer id found in IdeaTree UserProfile id:"+str(user_profile.id)) cancelAccountPart2(request, stripecustomerid=stripecustomerid, paidAccount=paidAccount) # ------------------------- THE CRUX OF IT -------------------------------------------- return signout(request) except IntegrityError as err: print("Cancel account error: "+str(err), file=sys.stderr) return HttpResponse("Cancel account error: "+str(err), status=406) except stripe.error.InvalidRequestError as err: print("Cancel account error: "+str(err), file=sys.stderr) return HttpResponse("Cancel account error: "+str(err), status=200) except IntegrityError as err: print("Cancel account error: "+str(err), file=sys.stderr) return HttpResponse("Cancel account error: "+str(err), status=406) except Exception as err: print("Cancel account error: "+str(err), file=sys.stderr) return HttpResponse("Cancel account error: "+str(err), status=406) @login_required() @user_passes_test(not_guest_user) def cancelAccountPart2(request, stripecustomerid=None, paidAccount=False): try: # FIXME: the workflow is not ideal here, since finishSuspenAccount() has two access points, directly here or via webhook(), but # I don't want to put all the eggs re: free trial in Stripe's basket, but rather handle it myself, in case some other payment # processor is chosen later. if paidAccount and stripecustomerid: stripe.api_key = settings.STRIPE_API_KEY customer = json.loads(str(stripe.Customer.retrieve(stripecustomerid))) if stripecustomerid == customer['id']: # One last check comparing the Stripe customerid stored on IdeaTree with the one stored at Stripe. # This will cause Stripe to send an 'event' to webhook() if customer['subscriptions']['total_count'] > 1: return HttpResponse("Error: multiple subscriptions found. Please contact your administrator. Reference: your IdeaTree id is: "+ customer['metadata']['user_id'], status=406) elif customer['subscriptions']['total_count'] == 0: raise Exception("No Stripe subscription found. Reference: Stripe customer ID: "+ customer['id']) else: subscriptionid = customer['subscriptions']['data'][0]['id'] stripe.Subscription.delete(subscriptionid, prorate=True, invoice_now=True) else: raise Exception("cancelAccountPart2: mismatched Stripe customer Ids.") else: # Webhook also calls the following if the above branch is taken: cancelAccountPart3(request) except IntegrityError as err: raise except stripe.error.InvalidRequestError as err: raise except Exception as err: raise def cancelAccountPart3(request, event=None): try: userId = int(request.session["_auth_user_id"]) if "_auth_user_id" in request.session else None if userId: user = User.objects.get(pk=userId) userProfile = UserProfile.objects.get(user=user) # If we weren't provided a user id directly, attempt to infer it from the Stripe customer ID. elif event: eventCustomer = event.data.object.customer userProfile = UserProfile.objects.get(stripecustomerid=eventCustomer) user = userProfile.user userId = user.id else: raise Exception("cancelAccountPart3: Not enough data in either request or event to determine user Id.") # FIXME: write test that every dependent on User is cascade-deleted from the db after cron-job garbage collection of User. with transaction.atomic(): user.is_active=False user.save() userProfile.accounttype=settings.FREE_ACCT userProfile.save() # NOTE: relies on db constraints to delete nodes, edges, comments, etc.: Map_desc.objects.filter(owner=user).update(status=settings.MAP_DELETED) if userProfile.deleteglobally: print("Deleting global data.", file=sys.stderr) # FIXME URGENT: test that user data in others' maps is marked for deletion, and only theirs. Node.objects.filter(owner=user).update(status=settings.NODE_DELETED) NodeComment.objects.filter(user=user).update(status=settings.NODE_COMMENT_DELETED) Edge.objects.filter(owner=user).update(status=settings.EDGE_DELETED) else: print("No global deleting will be done.", file=sys.stderr) print("Successfully cancelled account of user: "+str(user) + " id:"+str(user.id), file=sys.stderr) except Exception as err: # Poor man's error logging. If called from the webhook, which has already gone on to completion, there's no place to return an exception. print("CancelAccountPart3: "+str(err), file=sys.stderr) raise Exception(err) def upgradeSuccess(request, checkoutSessionId=None): # a) checkoutSessionId is sent here for an optional double-check: # This success page can parse checkoutSessionId out and fetch it using the Stripe API and get the real state of the CheckoutSession and its Subscriptions. # Then this can be coupled with webhook events to build resiliency in the case where the customer closes out of Checkout after paying (but just before # the success_url redirect), in which case your webhook event would still hear about this checkoutSession completing. # b) Also, it's possible to query Stripe for the results of event delivery and build a report for users. See https://stripe.com/docs/api/events/list#list_events-delivery_success # c) Another way to provide the user with a list of payments via an API call to Stripe, and parsing and formatting the results: # See https://stripe.com/docs/api/payment_intents/list#list_payment_intents-customer try: return render(request,'ideatree/upgradeSuccess.html') except Exception as err: return HttpResponse(str(err), status=406) # Docs: https://stripe.com/docs/payments/checkout/subscriptions/ # https://stripe.com/docs/payments/checkout/set-up-a-subscription # https://python.plainenglish.io/build-a-payment-system-with-django-3-1-and-stripe-from-scratch-part-1-21e8f980a9eb @login_required() def upgrade(request): try: userId = int(request.session["_auth_user_id"]) user = User.objects.get(pk=userId) userProfile = UserProfile.objects.get(user_id=user) stripeCustomerId = userProfile.stripecustomerid stripe.api_key = settings.STRIPE_API_KEY # ------------------------------------- THE CRUX OF IT ----------------------------------------------- stripesession = stripe.checkout.Session.create( success_url="https://www.ideatree.net/ideatree/upgradeSuccess/{CHECKOUT_SESSION_ID}/'", cancel_url="https://www.ideatree.net/ideatree/map/", # will reroute to landing page if user not logged in payment_method_types=['card'], client_reference_id=userId, line_items=[{ 'price': settings.STRIPE_PRICE_ID, 'quantity': 1, }], customer = stripeCustomerId, mode='subscription' ) # Redirect to the URL returned on the session return HttpResponseRedirect(stripesession.url, status=303) #context = {'stripesessionid':stripesession.id, 'ideatree_stripe_account':settings.STRIPE_IDEATREE_ACCOUNT_ID } #return render(request,'ideatree/upgrade.html', context) # FIXME: is this template obsolete? # FIXME errors returned from Stripe are not being handled. except IntegrityError as err: return HttpResponse(str(err), status=406) except Exception as err: return HttpResponse(str(err), status=406) # NOTE: For production, in order to receive events this webhook must be registered at https://dashboard.stripe.com/webhooks # NOTE: To set up a realistic event similar to how it will be in production, try tweaking the fixture at https://github.com/stripe/stripe-cli/blob/master/pkg/fixtures/triggers/checkout.session.completed.json # NOTE: It's possible to verify events using signatures (may use Webhook.construct_event instead of Event.construct_from). See https://stripe.com/docs/webhooks/signatures @csrf_exempt def stripe_webhook(request): import threading payload = request.body event = None try: try: event = stripe.Event.construct_from(json.loads(payload), stripe.api_key) except ValueError as e: # Invalid payload return HttpResponse(status=400) # Handle the event if event.type == 'checkout.session.completed': # Payment is successful and the subscription is created. # Provision the subscription and save the customer ID to IdeaTree's database. # NOTE: Since Stripe wants a reply quickly or they'll re-send, this would normally be done in a separate, asynchronous thread. # But by doing it inline, if there's a problem, Stripe will be notified that the event failed. Only three db accesses here, so it's fast enough. checkout_session = event['data']['object'] # Make sure is already paid and not delayed if checkout_session.payment_status == "paid": # Update db with paid status and increase their capacities # Some cross checks stripecustomerid = checkout_session.get("customer") stripeClientReference = checkout_session.get("client_reference_id") stripeClientReference = int(stripeClientReference) if stripeClientReference else None userProfile = UserProfile.objects.get(stripecustomerid=stripecustomerid, user_id=stripeClientReference) # NOTE: currently email is not cross-referenced because If the customer changes their email on the Checkout page, Stripe's Customer object # will be updated with the new email (which may no longer match the email that IdeaTree has for that customer). # This is as of Oct. 2021, but Stripe is working on changing that. https://stripe.com/docs/api/checkout/sessions/create#create_checkout_session-customer newVals = { 'stripecustomerid':stripecustomerid, 'accounttype':settings.REGULAR_ACCT, 'nummapsallowed':settings.PREMIUM_USER_NUM_MAPS_ALLOWED } for key, value in newVals.items(): setattr(userProfile, key, value) userProfile.save() print("Successful IdeaTree-side handling of Checkout payment for Stripe customerid: "+ stripecustomerid +". Updated user with: "+str(newVals), file=sys.stderr) #elif event.type == 'invoice.paid': # Continue to provision the subscription as payments continue to be made. # Store the status in your database and check when a user accesses your service. # FIXME: TODO #pass #elif event.type == 'invoice.payment_failed': # The payment failed or the customer does not have a valid payment method. # The subscription becomes past_due. Notify your customer and send them to the # customer portal to update their payment information. # FIXME: TODO #pass elif event.type == 'customer.subscription.deleted': th = threading.Thread(target=cancelAccountPart3, args=(request,event)) th.start() else: # Unhandled event. pass return HttpResponse(status=200) except ValueError as err: #return SuspiciousOperation(err) FIXME HttpResponse("SuspiciousOperation: " + str(err), status=405) except stripe.error.SignatureVerificationError as err: #return SuspiciousOperation(err) FIXME return HttpResponse("SuspiciousOperation: " + str(err), status=405) except UserProfile.DoesNotExist as err: return HttpResponse(str(err), status=406) except IntegrityError as err: return HttpResponse(str(err), status=406) except stripe.error.InvalidRequestError as err: return HttpResponse(str(err), status=406) except Exception as err: return HttpResponse(str(err), status=406) # --------------------------------- END PAYMENT PROCESSING ------------------------------------------------ @csrf_protect def demoMap(request): try: from django.contrib.auth import authenticate, login from django.shortcuts import redirect # NOTE: after much trying to get an authentication to persist over a redirect boundary, here's how you do it: # https://docs.djangoproject.com/en/2.1/topics/auth/default/#auth-web-requests guestuser = authenticate(request, username=settings.GUEST_USERNAME, password=settings.GUEST_PASSWORD) if guestuser is None: raise Exception("Failed login for guest user.") login(request, guestuser) # NOTE: combining these queries into one with update_or_create() would introduce the potential of multiple demo maps if just one parameter is inadvertently changed. demomap = Map_desc.objects.filter(mapname=settings.DEMO_MAPNAME, owner=guestuser)[:1] if not demomap: demomap = Map_desc.objects.create(mapname=settings.DEMO_MAPNAME, owner=guestuser, description="Scratch pad "+settings.WHAT_A_GRAPH_IS_CALLED) #mapId = demomap.id mapId= demomap.values_list('id',flat=True)[0] request.session["mapId"] = mapId nodesleft = demomap.values_list('nodesleft',flat=True)[0] #nodesleft = demomap.nodesleft if not nodesleft: Map_desc.objects.filter(pk=mapId).update(nodesleft=settings.MAX_FREE_ACCOUNT_NODES_PER_MAP) form = OpenMapForm({"mapId":mapId}) if form.is_valid(): return HttpResponseRedirect('/ideatree/map/') else: return HttpResponse("Invalid entry:" + str(form.errors),status=406) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) except Exception as err: return HttpResponse("System error:" + str(err), status=406) demoMap.alters_data = True #from wsgitrace import wsgiPdb #result = wsgiPdb.Debugger(demoMap) #@login_required() # FIXME #def backup(request): #return render(request,'ideatree/backup.html') def commandSummary(request): try: context = {"whatnodeiscalled":settings.WHAT_A_NODE_IS_CALLED, "whataclusteriscalled":settings.WHAT_A_CLUSTER_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/commandSummary.html', context) except Exception as err: return HttpResponse(str(err), status=406) def quickStartHelp(request): try: context = {"whatnodeiscalled":settings.WHAT_A_NODE_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/quickStart.html', context) except Exception as err: return HttpResponse(str(err), status=406) def prettifyHelp(request): try: context = {"whatnodeiscalled":settings.WHAT_A_NODE_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/prettify.html', context) except Exception as err: return HttpResponse(str(err), status=406) def prettifyCircularHelp(request): try: context = {'whataclusteriscalled':settings.WHAT_A_CLUSTER_IS_CALLED, 'whatanodeiscalled': settings.WHAT_A_NODE_IS_CALLED, 'what_an_edge_is_called':settings.WHAT_AN_EDGE_IS_CALLED } return render(request,'ideatree/help/prettifyCircular.html', context) except Exception as err: return HttpResponse(str(err), status=406) def prettifyRadialHelp(request): try: context = {'whataclusteriscalled':settings.WHAT_A_CLUSTER_IS_CALLED, 'whatanodeiscalled': settings.WHAT_A_NODE_IS_CALLED, 'what_an_edge_is_called':settings.WHAT_AN_EDGE_IS_CALLED } return render(request,'ideatree/help/prettifyRadial.html', context) except Exception as err: return HttpResponse(str(err), status=406) def prettifySpringHelp(request): try: context = {'whataclusteriscalled':settings.WHAT_A_CLUSTER_IS_CALLED, 'whatanodeiscalled': settings.WHAT_A_NODE_IS_CALLED, 'what_an_edge_is_called':settings.WHAT_AN_EDGE_IS_CALLED } return render(request,'ideatree/help/prettifySpring.html', context) except Exception as err: return HttpResponse(str(err), status=406) def clusteringHelp(request): try: context = {"whatnodeiscalled":settings.WHAT_A_NODE_IS_CALLED, "whataclusteriscalled":settings.WHAT_A_CLUSTER_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/clustering.html', context) except Exception as err: return HttpResponse(str(err), status=406) def teamsHelp(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/teams.html', context) except Exception as err: return HttpResponse(str(err), status=406) def notifyTeamHelp(request): try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/notifyTeam.html', context) except Exception as err: return HttpResponse(str(err), status=406) def edgeWeightHelp(request): try: # FIXME: 'weight' is now called 'cost' to differentiate from graphviz edge weight. context = {'edge_default_weight':settings.DEFAULTEDGE_COST, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} # FIXME: rename template file to help/edgeCosting.html return render(request,'ideatree/help/edgeWeighting.html', context) except Exception as err: return HttpResponse(str(err), status=406) def conceptVsMindHelp(request): try: context = {"whatnodeiscalled":settings.WHAT_A_NODE_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/conceptVsMindMapping.html', context) except Exception as err: return HttpResponse(str(err), status=406) def exportHelp(request): try: context = {"whatnodeiscalled":settings.WHAT_A_NODE_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/help/exporting.html', context) except Exception as err: return HttpResponse(str(err), status=406) def saveExplain(request): try: # Because this came from JS, it results in a JS popup. return render(request, "saveExplain.html") except Exception as err: return HttpResponse(str(err), status=406) def nodeShapes(getIndexFor=False): try: shapes = settings.NODE_SHAPES if getIndexFor: return shapes.index(getIndexFor) return shapes except IndexError as err: raise Warning(str(err)) except TypeError as err: raise Warning(str(err)) def lookup_user_info(userID,field): try: # FIXME: why select_related if it's getting info from the Profile itself and not from the related User? # FIXME: is this efficient? Does it get all rows before filtering by pk? Why selected_related? return UserProfile.objects.select_related(field).values_list(field,flat=True).get(user__id=userID) except: raise lookup_user_info.alters_data = True # FIXME: change to return a dict, with multiple values from one query. def lookup_node_info(nodeId,field): try: #return Node.objects.select_related(field).values_list(field,flat=True).get(pk=nodeId) info = Node.objects.values_list(field,flat=True).get(pk=nodeId) # FIXME: is this efficient? Does it get all rows before filtering by pk? if info or info==0: return info else: return None except: raise def lookup_node_by_field(fieldname, value): try: #return Node.objects.select_related(field).values_list(field,flat=True).get(pk=nodeId) info = Node.objects.values(fieldname,flat=True).filter(fieldname=value) # FIXME: is this efficient? Does it get all rows before filtering by pk? if info: return info else: return None except: raise # FIXME: rename: get_map_info def lookup_map_info(mapId,field): try: # FIXME: is this efficient? Does it get all rows before filtering by pk? return Map_desc.objects.values_list(field,flat=True).get(pk=mapId) # FIXME: more precise Exception except Exception: raise def set_map_property(mapId,fieldname,fieldval): try: Map_desc.objects.filter(id=mapId).update(**{ fieldname: fieldval }) return True except Exception: raise def friends_of(userID): try: return Friend.objects.filter(initiator=userID, status=settings.FRIEND_ACCEPTED).values('friend_id') except Exception: raise def friends_invited(userID): # FIXME: obsolete try: return Friend.objects.filter(user_id=userID, status=settings.FRIEND_INVITED).friend_id except Exception: raise def map_membership(userID, mapId): try: return Mapmember.objects.filter(member_id=userID, ofmap_id=mapId) except Exception: raise # FIXME: use this everywhere there is now a check for an open map. def currently_opened_map(request): try: if request.session.get('mapId'): return int(request.session['mapId']) # int() is cheap validation #FIXME good enough? else: raise Warning(settings.PLEASE_OPEN_GRAPH_PROMPT) except: raise # FIXME: try/catch for all these util defs # FIXME test everywhere it occurs # FIXME: there's some place on map open where this doesn't raise a prompt def have_map_access(request, mapId): try: update_mymap_list(request) # expensive, but safe, to do this every time. mapIdList = [] for mapDict in json.loads(request.session["accessible_maps"]): mapIdList.append(mapDict['id']) if not int(mapId) in mapIdList: # FIXME: provide a return button in a template. raise PermissionDenied(settings.PERMISSION_DENIED_PROMPT) except: raise have_map_access.alters_data = True # FIXME: notify and crudNodeComments need this (see DRY comment) but need mapmember ids returned, not mapmember.member ids def all_map_members(mapId, idsOnly=False, asObjects=False): try: users = Mapmember.objects.filter(ofmap=mapId, status=settings.MAPMEMBERSHIP_ACTIVE) userList = [] for user in users: if idsOnly: userList.append(user.member.id) elif asObjects: userList.append(user) else: userList.append({user.member.id:user.member.username}) return userList except Exception: raise all_map_members.alters_data = True def have_relationship(userID1, userID2): # FIXME: dead code try: return ((userID1 in friends_of(userID2)) or (userID2 in friends_of(userID1)) ) except Exception: raise def duplicate_mapname(mapname): try: return Map_desc.objects.filter(mapname=mapname, status=settings.MAP_ACTIVE).exists() except Exception: raise duplicate_mapname.alters_data = True def all_nodes_in_this_map(mapId, orderBy='created_date', countOnly=False, excludeClusters=False): try: # FIXME: use one query with additional filters for each condition. if countOnly: return Node.objects.filter(ofmap__id=mapId).exclude(status=settings.NODE_DELETED).exclude(nodetype=settings.PROVISIONAL).count() else: if excludeClusters: return Node.objects.filter(ofmap__id=mapId).exclude(nodetype=settings.CLUSTER).exclude(status=settings.NODE_DELETED).exclude(nodetype=settings.PROVISIONAL).order_by(orderBy) else: return Node.objects.filter(ofmap__id=mapId).exclude(status=settings.NODE_DELETED).exclude(nodetype=settings.PROVISIONAL).order_by(orderBy) except Exception: raise all_nodes_in_this_map.alters_data = True # FIXME: remove #def showusers(request): #try: #allusers = User.objects.all() #for user in allusers: #print(user.username,' ',user.email) #return HttpResponse("OK") #except Exception: #raise def all_accessible_maps(request, thisUser, exclude_current=False ): # Maps I have access to, whether owned by me or others # FIXME: this is called MULTIPLE times when just opening a map. try: # NOTE: see my question and an answer on StackOverflow: https://stackoverflow.com/questions/48659202/how-do-you-do-multiple-joins-simultaneously-in-django-orm/48659313#48659313 #sharedmaps = Mapmember.objects.filter(member=thisUser,status=settings.MAPMEMBERSHIP_ACTIVE).values_list('ofmap') #allaccessible = Map_desc.objects.filter(Q(id__in=sharedmaps) | Q(owner=thisUser,status=settings.MAP_ACTIVE)) # NOTE: 'mapmember' looks into Mapmember table. allaccessible = Map_desc.objects.filter(Q(owner=thisUser,status=settings.MAP_ACTIVE) | Q(mapmember__member=thisUser,mapmember__status=settings.MAPMEMBERSHIP_ACTIVE)).distinct().order_by('mapname') if exclude_current: return(allaccessible.exclude(pk=int(request.session["mapId"]))) else: return(allaccessible) except: raise all_accessible_maps.alters_data = True def update_mymap_list(request): try: thisUser = User.objects.get(pk=int(request.session["_auth_user_id"])) #FIXME: removed mapmembership status field from db. ok? #FIXME: garbage collect friends 'D' status from db, and change prompt. # FIXME: test that you can't add a user to mapmembership table if they don't have_relationship() with owner of the map request.session["accessible_maps"] = json.dumps(list(all_accessible_maps(request, thisUser).values('id','mapname','description','owner'))) # Subset of session["accessible_maps"] request.session["owned_maps"] = json.dumps(list(Map_desc.objects.filter(owner=thisUser, status=settings.MAP_ACTIVE).values('id','mapname','description'))) except: raise update_mymap_list.alters_data = True # FIXME: should also remove directories? If so, remove_absoluteOutfileDir() is not needed def clearOutputFiles(outputDirName, skipExtension=None): # # FIXME UPDATE: some of this not true since file caching isn't used anymore. # Erases graph files which triggers a re-rendering from the database when layout is requested. # Here we rely on unique-to-a-process filenames, and on Graphutils::eraseOutputFiles() not deleting any .fbc file, # in order for this output to stay around long enough to be sent to Firebase. # It must be explicitly deleted by the process which kicked this off. # Map image files, (.json), are handled differently, since they hang around in the file system and can be accessed # by a client using only the web server, not going through this python code. # # @param integer outputDirName identifies the directory where the map(graph) files are # @author Ron Newman # @see graphRender() # do NOT erase directories, since embedded maps and exports are there try: filelist = glob.glob(str(outputDirName) + "*.*") for filename in filelist: extension = os.path.splitext(filename)[1].lower() if((extension != "fbc") and # used by mapLayout to queue change messages for firebase. It will handle deletions. (extension != "gxl") and # used by GraphRender and by gxl import, which will delete it itself. (extension != "xdot") and # used by GraphRender, which will delete it itself. (extension != "gv") and # used by GraphRender and Import, which will delete it themselves. (skipExtension and extension != skipExtension)): os.unlink(filename) except Exception: raise clearOutputFiles.alters_data = True # FIXME: not currently used. No backups provided, only exports. def eraseMapBackupDirs(userID): # FIXME: write test of this # # erases graph backups # # Depends on the fact that the original creation of the zip file (Make_backup.php5) # erases all the source data after the compressed file is completed. If that # is changed, then there's a recursive unlink routine available in the comments of the # php manual for the unlink() function and that should be used here. # # @param integer userId identifies the directory where the map(graph) files are # @author Ron Newman # try: mainBackupDir = settings.MAP_DOWNLOAD_DIR + "/user" + str(userID) if(os.path.isdir(mainBackupDir)): shutil.rmtree(mainBackupDir) except: raise eraseMapBackupDirs.alters_data = True def eraseExports(mapId): try: mainExportDir = make_absoluteExportDir(mapId) if(os.path.isdir(mainExportDir)): shutil.rmtree(mainExportDir) except: raise eraseExports.alters_data = True def shortLabel(label, labelLength): try: return (label[:labelLength] + '...') if len(label) > (labelLength + 3) else label except Exception: raise # -------------- helper functions for both map output, layout, and export -------------- # FIXME all of these used? # -------------- Dir get()s ---------------------------- def get_relativeOutfileDir(mapId): try: return(str(mapId) + '/') except Exception: raise def get_relativeOutfilePath(mapId, suffix): try: return (get_relativeOutfileDir(mapId) + get_outfileName(mapId, suffix)) except Exception: raise def get_outfileURL(mapId): try: return ('pull/'+ str(mapId) + '/') except Exception: raise def get_absoluteOutput_BASE_Dirname(): return(settings.BASE_DIR + '/' + settings.OUTPUTDIRNAME + '/') # return an output directory path suitable for web access via urls.py def get_absoluteOutfileDir(mapId): try: return (get_absoluteOutput_BASE_Dirname() + get_relativeOutfileDir(mapId)) except Exception: raise def get_absoluteOutfilePath(mapId, suffix): try: outputDir = get_absoluteOutfileDir(mapId) return(outputDir + get_outfileName(mapId, suffix)) except Exception: raise def get_absoluteTemplatesDir(): try: return(settings.TEMPLATES[0]['DIRS'][0] + '/') except Exception: raise def get_absoluteExportPath(mapId, suffix): try: exportDir = make_absoluteExportDir(mapId) return(exportDir + get_outfileName(mapId, suffix)) except Exception: raise def get_outfileName(mapId, suffix=settings.MAPLOAD_TEMP_FILE_SUFFIX): try: # FIXME: use separate filenames for outline and graphic return (str(mapId) + "." + suffix) except Exception: raise def make_absoluteOutfilePath(mapId, suffix): try: return(make_absoluteOutfileDir(mapId) + get_outfileName(mapId, suffix)) except Exception: raise # -------------- directory make()s---------------------------- def make_publishPath(mapId, suffix, absolute=True): try: if absolute: pubDir = make_absolutePublishDir(settings.PUBLISH_DIR_PATH, mapId) return(os.path.join(pubDir, str(mapId) + "." + suffix)) else: return(os.path.join(settings.PUBLISH_DIRNAME, str(mapId),str(mapId) + "." + suffix)) except Exception: raise # FIXME: rename. not necessarily absolute def make_absolutePublishDir(mainPublishDir, mapId): try: # FIXME: improve efficiency if the dirs below exist. os.makedirs(mainPublishDir,exist_ok=True) # NOTE: on some systems, mode is ignored on makedirs() so we set it explicitly # FIXME: can permissions here be more restrictive? myutils.doWSGIPermissions(mainPublishDir, mask=stat.S_IRWXU | stat.S_IRWXG ) # Make the second level directory, simply named by the map id: '/' mapSpecificPublishDir = os.path.join(mainPublishDir,str(mapId)) os.makedirs(mapSpecificPublishDir,exist_ok=True) # NOTE: on some systems, mode is ignored on makedirs() so we set it explicitly # FIXME: can permissions here be more restrictive? myutils.doWSGIPermissions(mapSpecificPublishDir, mask=stat.S_IRWXU | stat.S_IRWXG ) return(mapSpecificPublishDir) except Exception: raise # FIXME make this more efficient, rather than re-making two level directories every time, but watch out for race conditions. def make_absoluteOutfileDir(mapId): try: # Make the first level directory: '/' baseoutputDir = get_absoluteOutput_BASE_Dirname() # FIXME: change to run() userId = int(check_output(['id -u wsgiuser'],shell=True)) os.makedirs(baseoutputDir,exist_ok=True) # NOTE: on some systems, mode is ignored on makedirs() so we set it explicitly # NOTE: stat modes at https://www.tutorialspoint.com/python/os_chmod.htm # FIXME: use myutils.doWSGIPermissions(mapSpecificPublishDir, mask=stat.S_IRWXU ) os.chmod(baseoutputDir, stat.S_IRWXU) os.chown(baseoutputDir,userId, -1) # Make the second level directory, simply named by the map id: '/' second_level_output_dir = baseoutputDir + get_relativeOutfileDir(mapId) os.makedirs(second_level_output_dir,exist_ok=True) # NOTE: on some systems, mode is ignored on makedirs() so we set it explicitly os.chmod(second_level_output_dir, stat.S_IRWXU) os.chown(second_level_output_dir,userId,-1) return(second_level_output_dir) except: raise # FIXME: rename. nothing absolute about it def make_absoluteExportDir(mapId): try: exportDir = get_absoluteOutfileDir(mapId) + settings.EXPORTS_DIRNAME os.makedirs(exportDir, exist_ok=True) # NOTE: on some systems, mode is ignored on makedirs() so we set it explicitly # NOTE: stat modes at https://www.tutorialspoint.com/python/os_chmod.htm # See: https://stackoverflow.com/questions/273192/how-can-i-safely-create-a-nested-directory-in-python/14364249#14364249 os.chmod(exportDir, stat.S_IRWXU) return exportDir + "/" except Exception: raise def make_absoluteImportDir(mapId): try: # For fine points, see: https://stackoverflow.com/questions/273192/how-can-i-safely-create-a-nested-directory-in-python/14364249#14364249 # FIXME importDir = settings.MEDIA_ROOT + "/" + str(mapId) + "/" + settings.IMPORTS_DIRNAME importDir = get_absoluteOutfileDir(mapId) + settings.IMPORTS_DIRNAME os.makedirs(importDir, exist_ok=True) # NOTE: on some systems, mode is ignored on makedirs() so we set it explicitly # NOTE: stat modes at https://www.tutorialspoint.com/python/os_chmod.htm # See: https://stackoverflow.com/questions/273192/how-can-i-safely-create-a-nested-directory-in-python/14364249#14364249 os.chmod(importDir, stat.S_IRWXU) return importDir except: raise # ---------------- Misc dir functions ------------------------------------------ def remove_absoluteOutfileDir(mapId): try: outputDir = get_absoluteOutfileDir(mapId) if(os.path.isdir(outputDir)): #FIXME has a race condition. Look for atomic alternative. shutil.rmtree(outputDir) # does it have a 'not_exists_ok' option? except: raise def clear_absoluteExportDir(mapId): try: exportDir = make_absoluteExportDir(mapId) filelist = glob.glob(exportDir + "/*.*") for file in filelist: os.unlink(file) except Exception: # FIXME: should be file-specific exception, or just except: ? raise clear_absoluteExportDir.alters_data = True def set_map_access_changed(user): # # set database flag to denote that access to some map has changed for a given user. # # @author Ron Newman try: Map_access_changed.objects.update_or_create(changed=True, user=user) except: raise set_map_access_changed.alters_data = True @user_passes_test(not_guest_user) @transaction.atomic() def makeMap(request, mapname=None, description="A starter "+settings.WHAT_A_GRAPH_IS_CALLED+"."): try: mapMetrics = _mapsAllowedMetrics(request) if (_mapNumLimitReached(request, numMapsAllowed=mapMetrics['numMapsAllowed'], ownedMaps=mapMetrics['ownedMaps'])): raise Exception("Allowable number of "+settings.WHAT_A_GRAPH_IS_CALLED+"s exceeded.") userID = int(request.session["_auth_user_id"]) if(not mapname): mapname=request.session["username"] + "'s starter "+settings.WHAT_A_GRAPH_IS_CALLED # FIXME: what is "editing_map" for?: # if(empty($_POST["editing_map"]) && if (duplicate_mapname(mapname)): prompt = "Sorry, '"+ mapname +"' is taken. Please select another "+settings.WHAT_A_GRAPH_IS_CALLED+" name." raise Warning(prompt) # FIXME: what is "editing_map" for?: if(not request.session.get("editing_map") or not request.get("editing_map")): if lookup_user_info(userID,"accounttype")==settings.FREE_ACCT: allowedNodes=settings.MAX_FREE_ACCOUNT_NODES_PER_MAP else: allowedNodes=settings.MAX_PREMIUM_ACCOUNT_NODES_PER_MAP thisUser = User.objects.get(pk=userID) newmap = Map_desc.objects.create(mapname=mapname, owner=thisUser, nodesleft=allowedNodes, description=description) # FIXME status no longer used: "status"=>'V'); Mapmember.objects.create(ofmap=newmap, member=thisUser) # because when moving to a new system, old output files can exist and be displayed when the old map ID corresponds to a new map ID. clearOutputFiles(get_absoluteOutfileDir(newmap.id)) # used when this function called in the process of doing layout or export set_map_access_changed(User.objects.get(pk=userID)) #eraseMapBackupDirs(userID) update_mymap_list(request) return newmap.id except: raise makeMap.alters_data = True def _mapsAllowedMetrics(request): try: userId = int(request.session["_auth_user_id"]) numMapsAllowed = UserProfile.objects.get(user__id=userId).nummapsallowed ownedMaps = json.loads(request.session["owned_maps"]) if request.session.get("owned_maps") else [] return({'numMapsAllowed':numMapsAllowed, 'ownedMaps':ownedMaps}) except: raise def _mapNumLimitReached(request, numMapsAllowed, ownedMaps): noMoreAllowed = ("is_student" in request.session) or (request.session["username"] == settings.GUEST_USERNAME) or (len(ownedMaps) >= numMapsAllowed) return(noMoreAllowed) @login_required() @csrf_protect def newMap(request): try: if not request.POST.get('submitted'): update_mymap_list(request) mapMetrics = _mapsAllowedMetrics(request) numMapsAllowed = mapMetrics['numMapsAllowed'] ownedMaps = mapMetrics['ownedMaps'] noMoreAllowed = _mapNumLimitReached(request, numMapsAllowed=numMapsAllowed, ownedMaps=ownedMaps) form = NewMapForm() context = { 'form':form, 'numMapsAllowed':numMapsAllowed, 'ownedMaps':ownedMaps, 'noMoreAllowed':noMoreAllowed, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } return render(request, "newmap.html", context, content_type="application/html") else: form = NewMapForm(request.POST) if form.is_valid(): mapId = makeMap(request, form.cleaned_data['mapname'], form.cleaned_data['description']) context = {'form':form, 'mapId':mapId, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } return render(request, "newmapSuccess.html", context, content_type="application/html") except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) newMap.alters_data = True @user_passes_test(not_guest_user) @transaction.atomic() @login_required() @csrf_protect def deleteMap(request): # Delete all aspects of a map from the database, membership, description, etc. # NOTE: assumes all nodes of a map have been deleted, so it does nothing to node-related tables. Keep it that way for safety. # @author Ron Newman try: mapId = currently_opened_map(request) userId = int(request.session["_auth_user_id"]) have_map_access(request, mapId) # FIXME: possible to write these preliminaries as a decorator? ownedMaps = json.loads(request.session["owned_maps"]) owned_list_of_ids = [m['id'] for m in ownedMaps] if not mapId in owned_list_of_ids: raise Warning(settings.MUST_OWN_GRAPH_PROMPT) numnodes = all_nodes_in_this_map(mapId, countOnly=True) if numnodes: raise Warning(settings.DELETE_AFTER_GRAPH_EMPTY_PROMPT) mapname = [n["mapname"] for n in ownedMaps if n["id"] == mapId] context = {'mapname':mapname[0], 'what_nodes_are_called':settings.WHAT_A_NODE_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } if not request.POST.get('submitted'): # Since there is no user-entered data, we don't use a form for validation, THE INITIAL TIME AROUND (but do use it after the form is created and submitted). return render(request, "deleteMap.html", context, content_type="application/html") else: form = DeleteMapForm(request.POST) if form.is_valid(): # mainly to force a check of csrf_token # Do the deletions. # Remove everyone else's access to this map for user in all_map_members(mapId,False,True): # Since this map soon won't exist anymore, alter everyone's access to it. Do this before the deletion. set_map_access_changed(user.member) # Mark records as deleted for garbage collection Map_desc.objects.filter(pk=mapId).update(status=settings.MAP_DELETED) Mapmember.objects.filter(ofmap=mapId).update(status=settings.MAPMEMBERSHIP_DELETED) eraseExports(mapId) # empty directory that holds graph exports clearOutputFiles(get_absoluteOutfileDir(mapId)) # empty the directory that holds graph info remove_absoluteOutfileDir(mapId) #eraseMapBackupDirs(userId) del request.session["mapId"] return render(request, "deleteMapSuccess.html", context, content_type="application/html") else: context.update({'form':form}) return render(request, "deleteMap.html", context, content_type="application/html") except PermissionDenied as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) deleteMap.alters_data = True def signout_firebase(request): # FIXME TODO return(True) def signout(request): try: signout_firebase(request) return HttpResponseRedirect('/accounts/logout/') except: raise @user_passes_test(not_guest_user) @transaction.atomic() @login_required() @csrf_protect def editMapName(request): try: mapId = currently_opened_map(request) userId = int(request.session["_auth_user_id"]) try: thisMapOwnedByThisUser = Map_desc.objects.get(pk=mapId, owner_id=userId) except Map_desc.DoesNotExist: raise Warning("Sorry, you must be the original creator of this "+settings.WHAT_A_GRAPH_IS_CALLED+" to change its name.") context = {} submitted = request.POST.get("submitted") submitted=True if submitted in ("1",'true',"True") else False if not submitted: mapname = request.session["mapname"] if (request.session.get("mapname")) else None # previous, existing mapname description = request.session["mapDescription"] if (request.session.get("mapDescription")) else None #previous, existing description form = NewMapForm(initial={'mapname':mapname, 'description':description} ) context = {'form':form, 'mapname':mapname, 'description':description, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request, "editMapnameAndDescription.html", context, content_type="application/html") else: instance = Map_desc.objects.get(pk=mapId) # to enforce updating rather than creation of a new map. form = NewMapForm(request.POST,instance=instance) if form.is_valid(): newFormName = form.save(commit=False) newFormName.owner = User.objects.get(pk=userId) newFormName.save() updateMapSessionData(request,mapId) context = { "mapname":request.session["mapname"],"mapId":mapId, "mapDescription":request.session["mapDescription"] } return render(request, "editMapnameSuccess.html", context, content_type="application/html") else: context.update({'form':form, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED}) return render(request, "editMapnameAndDescription.html", context, content_type="application/html") except IntegrityError as err: # for form.save() return HttpResponse(err, status=406) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) editMapName.alters_data = True def get_truncated_mapname(request,mapId): try: length = settings.MAPNAME_TRUNCATED_LENGTH for mapDict in json.loads(request.session["accessible_maps"]): if int(mapDict['id'])==int(mapId): mapname=mapDict["mapname"] mapname = (mapname[:length] + '...') if len(mapname) > (length+3) else mapname return(mapname) return "[unknown map]" except: raise def freeTrialDates(auser, nowTime): try: # NOTE: 'registerdate' must be ensured non-null in the model registerdate = lookup_user_info(auser.id,"registerdate") trialPeriodDays = lookup_user_info(auser.id,"trialperioddays") trialEndWarningTime=registerdate + timedelta(days=trialPeriodDays) trialEndTime=registerdate + timedelta(days=trialPeriodDays + settings.FREE_ACCT_GRACEPERIOD_DAYS) timeToExpiration = relativedelta(trialEndTime, nowTime) return({'trialEndWarningTime':trialEndWarningTime, 'timeToExpiration':timeToExpiration}) except: raise # Step 1 of opening a map, called from dropdown menu. Returns a list of maps available to open. @login_required() #@csrf_protect #FIXME URGENT def openMap(request): # rename listMapsForSelect try: update_mymap_list(request) #FIXME second half of this statement really needed? json.loads needed? if not "owned_maps" in request.session or not json.loads(request.session["owned_maps"]): request.session["owned_maps"]=str([]) if 'readOnly' in request.session: signout() # get a clean session raise Warning('Open Map not available for read only accounts.') userID = str(request.session["_auth_user_id"]) thisUser = User.objects.get(pk=userID) # FIXME: test that only accessible maps are listed # they have no record of maps at all or map list is empty if not 'accessible_maps' in request.session or not json.loads(request.session["accessible_maps"]): makeMap(request) # FIXME: to be consistent, use 'submitted' as it is elsewhere # FIXME: use an empty form here for validation? context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} if not "mapId" in request.POST: showExpirationPrompt = False if lookup_user_info(thisUser.id,"accounttype")==settings.FREE_ACCT: nowTime = timezone.now() # NOTE: depends on TZ=True in settings.py!! ftd = freeTrialDates(thisUser, nowTime) # FIXME: put guest in global exempt users array if(nowTime >= ftd["trialEndWarningTime"] and request.session.get("username") != settings.GUEST_USERNAME): showExpirationPrompt=True; daysToExpiration = 0 if (ftd["timeToExpiration"].days < 0) or (ftd["timeToExpiration"].years < 0) else ftd["timeToExpiration"].days context.update({'daysToExpiration': daysToExpiration} ) owned_maps = json.loads(request.session["owned_maps"]) owned_list = [m['id'] for m in owned_maps] context.update({'owned_maps':owned_list, 'showExpirationPrompt':showExpirationPrompt, 'map_list':json.loads(request.session["accessible_maps"])}) # FIXME: pickmap.html has a js file at the bottom with incorrect path return render(request, "pickmap.html", context) else: form = OpenMapForm(request.POST) if form.is_valid(): # FIXME test # FIXME: shouldn't it be form.cleaned_data? have_map_access(request, form.data['mapId']) # FIXME: should this happen only after a successful pull? # FIXME: shouldn't it be form.cleaned_data? request.session["mapId"]=form.data['mapId'] thisMapHash= "#" + form.data['mapId'] + "/" # The following generates an onhashchange event which triggers a mapLoad() call based on thisMapHash return render(request, "pickmapResult.html", {'thisMapHash':thisMapHash}, content_type="application/html") else: return HttpResponse("Invalid entry:" + str(form.errors),status=406) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) except Exception as err: return HttpResponse("System error:" + str(err), status=406) openMap.alters_data = True @transaction.atomic() def updateMapSessionData(request, mapId): try: mapObj = Map_desc.objects.get(pk=mapId) request.session["mapDescription"] = mapObj.description request.session["mapname"] = mapObj.mapname request.session["map_owner"] = mapObj.owner.id request.session["map_owner_name"] = User.objects.get(pk=mapObj.owner.id).username request.session["mapId"]=mapObj.id except: raise updateMapSessionData.alters_data = True # FIXME: re-instate or delete def getPublishedDateDescription(mapId): try: publishedFilePath=settings.READ_ONLY_DIRECTORY_NAME + "/" + str(mapId) +"/public.json" if(os.path.isfile(publishedFilePath)): #return("PUBLISHED " + st_mtime(publishedFilePath).strftime("%a, %-d %b, %Y at %-I:%M %p")) return("PUBLISHED " + stat.st_mtime(publishedFilePath).strftime("%c")) else: return('') except: raise def makeTransactionHeaderJSON(mapId, transID=None, time=None): # FIXME: keep in python structures, not string. See Itree.php5::prepJSON try: # FIXME: originally, the check was for an object property: if(empty($this->transactionId)) if transID == None: randomNum = str(random.randrange(1000000)) transID ="API"+ randomNum; if time == None: time = str(myutils.millisNow()) if not mapId: raise Exception("Missing mapId.") # FIXME: make this actual JSON, not string, and close out changes. return('{"transactionId":"'+str(transID)+'", "time":'+str(time)+', "mapId":'+str(mapId)+', "changes":[') except: raise def makeSetMapAttributeChangeMessage(attrName, attrValue): try: CM = [] CM.append('setMapAttribute') # FIXME: accept dicts with multiple attributes if changeMessage format allows CM.append(attrName) CM.append(attrValue) return(CM) except: raise def makeSetNodeAttributeChangeMessage(nodeId, attrValue, attrName=None): try: CM = [] CM.append('setNodeAttribute') CM.append(nodeId) # FIXME: accept dicts with multiple attributes if changeMessage format allows if attrName: CM.append(attrName) CM.append(attrValue) return(CM) except: raise def makePutNodeInClusterChangeMessage(nodeId, clusterId): try: CM = [] CM.append('putNodeInCluster') CM.append(nodeId) CM.append(clusterId) return(CM) except: raise # FIXME: remove one of these duplicates def makeAlterEdgeChangeMessage(edgeId, paramName, paramValue, forGraphvizLayout=False): try: if forGraphvizLayout: return( (edgeId, edgeParams) ) # return a tuple: id,dict else: CM = [] CM.append("setEdgeAttribute") CM.append(edgeId) CM.append(paramName) CM.append(paramValue) return(CM) except: raise def makeSetEdgeAttributeChangeMessage(edgeId, attrName, attrValue): try: CM = [] CM.append('setEdgeAttribute') CM.append(edgeId) # FIXME: accept dicts with multiple attributes if changeMessage format allows CM.append(attrName) CM.append(attrValue) return(CM) except: raise def makeRemoveEdgeChangeMessage(edgeId, mapId): try: CM = [] CM.append('removeEdge') CM.append(edgeId) CM.append(mapId) return(CM) except: raise def makeCreateNodeChangeMessageJSON(nodeId, params, forGraphvizLayout=False): # FIXME: doesn't return JSON so change the name try: # FIXME: make nodetype a required argument, then derive pencolor, shape, style, fillcolor defaults from that. # nodestyle; // FIXME: when are we going to numeric values? #requiredArgs= {'ownerID':str(nodeOwnerID), 'pos':pos, 'timestamp':str(nodeDate)} # FIXME: change these two statements when Python 3.5+ is available #nodeargs = requiredArgs.copy() #params_dict.pop("ownerID") #params_dict.pop("creationDate") #params_dict.pop("pos") #params_dict.pop("targetmapinfo") # FIXME: used? #nodeargs.update(params_dict) # FIXME: again, hardcodes knowledge of change message format. if forGraphvizLayout: return( (nodeId, params ) ) # return a tuple: id,dict else: CM = [] CM.append("createNode") CM.append(nodeId) CM.append(params) return(CM) except: raise def makeCreateEdgeChangeMessageJSON(edgeId, edgeParams, forGraphvizLayout=False): # FIXME: doesn't return JSON so change the name try: if forGraphvizLayout: return( (edgeId, edgeParams) ) # return a tuple: id,dict else: CM = [] CM.append("createEdge") CM.append(edgeId) CM.append(edgeParams) return(CM) except: raise def makeGraphDefaults(request, mapId): # FIXME: rename makeGraphDefaults4Loading try: # FIXME: roll these into one query # FIXME: ensure that security has checked validity of mapId if not mapId: raise Exception("Missing mapId.") have_map_access(request, mapId) # FIXME re-implement or delete mapPublishInfo = getPublishedDateDescription(mapId); mapObj = Map_desc.objects.get(pk=mapId) mapname = mapObj.mapname map_description = mapObj.description map_bgcolor = mapObj.bgcolor # FIXME: remove strip() defaultNodeStyle = mapObj.defaultNodeStyle.strip() defaultNodeShape = mapObj.defaultNodeShape defaultNodeFillcolor = mapObj.defaultNodeFillcolor defaultNodeHeight = mapObj.defaultNodeHeight defaultNodeWidth = mapObj.defaultNodeWidth defaultNodeType = mapObj.defaultNodeType map_owner_name = Map_desc.objects.get(pk=mapId).owner.username # FIXME: make shape an index into a client-side table of names # FIXME: obsolete? if(empty($_SESSION["mobile"])) $defaultLabel=EMPTY_NODE_LABEL_PLACEHOLDER ; else $defaultLabel=""; # FIXME: some of these map attributes aren't actually accessed by the client. graphSpec = ' ["setMapAttribute", "mapname","'+ mapname +'"], ' graphSpec += ' ["setMapAttribute", "description","'+ map_description +'"], ' graphSpec += ' ["setMapAttribute", "mapOwner","'+ map_owner_name + '"], ' #graphSpec += ' ["setMapAttribute", "mapPublishInfo","'+ mapPublishInfo +'"], ' graphSpec += ' ["setMapAttribute", "bgcolor","'+ map_bgcolor +'"], ' graphSpec += ' ["setMapAttribute", "orientation","'+ settings.DEFAULT_MAP_LAYOUT_ORIENTATION +'"],' graphSpec += ' ["setMapAttribute", "defaultNodeOwner","'+ request.session["username"] +'"],' graphSpec += ' ["setMapAttribute", "defaultNodeStyle","'+ defaultNodeStyle +'"],' graphSpec += ' ["setMapAttribute", "defaultNodeShape",'+ str(defaultNodeShape) +'],' graphSpec += ' ["setMapAttribute", "defaultNodeFillcolor","'+ defaultNodeFillcolor +'"],' graphSpec += ' ["setMapAttribute", "defaultNodeHeight",'+ str(defaultNodeHeight) +'],' graphSpec += ' ["setMapAttribute", "defaultNodeWidth",'+ str(defaultNodeWidth) +'],' graphSpec += ' ["setMapAttribute", "defaultNodeType","'+ defaultNodeType +'"],' graphSpec += ' ["setMapAttribute", "defaultNodeLabel","'+ settings.EMPTY_NODE_LABEL_PLACEHOLDER +'"],' graphSpec += ' ["setMapAttribute", "currentUserID",'+ request.session['_auth_user_id'] +'],' # NOTE: in graphviz, pencolor is the border color, fontcolor is for text. graphSpec += ' ["setMapAttribute", "defaultNodePencolor","'+ settings.DEFAULTNODE_PENCOLOR +'"],' graphSpec += ' ["setMapAttribute", "defaultEdgeCost","'+ str(settings.DEFAULTEDGE_COST) +'"],' graphSpec += ' ["setMapAttribute", "defaultEdgeColor","'+ settings.DEFAULTEDGE_PENCOLOR +'"],' graphSpec += ' ["setMapAttribute", "defaultEdgeArrowhead",'+ str(settings.DEFAULTEDGE_ARROWHEAD_STYLE) +'],' graphSpec += ' ["setMapAttribute", "defaultEdgePenwidth","'+ str(settings.DEFAULTEDGE_PENWIDTH) +'"],' # FIXME $this->graphSpec .= " [\"setMapAttribute\", \"defaultNodeAttributes\",{ \"nodetype\":\"".ORIGINAL_NODE."\", \"defaultLabel\":\"".EMPTY_NODE_LABEL_PLACEHOLDER ."\", \"currentUserID\":".$_SESSION["userID"].", \"shape\":0, \"size\":\"".$this->defaultNodeWidth.",".$this->defaultNodeHeight."\", \"pencolor\":\"".$this->defaultNodePencolor."\", \"fillcolor\":\"".$this->defaultNodeFillcolor."\" }],"; # FIXME: why not use this? Seems a good thing. See additional question in makenodes(), next to 'pencolor' # graphSpec += ' ["setMapAttribute", "defaultPathendpointColor","'+ settings.PATH_ENDPOINT_NODE_FILLCOLOR +'"],' graphSpec += ' ["setMapAttribute", "defaultVotableLabel","'+ settings.EMPTY_NODE_LABEL_PLACEHOLDER +'"],' graphSpec += ' ["setMapAttribute", "defaultVotableFillcolor","'+ settings.VOTABLE_NODE_COLOR +'"],' graphSpec += ' ["setMapAttribute", "defaultVotableSliderParameters", {"value":5, "min":0, "max":10, "step":1, "submitName":"Send" }],' graphSpec += ' ["setMapAttribute", "defaultVotableSubmitName", "Send" ],' graphSpec += ' ["setMapAttribute", "defaultClusterType","'+ settings.CLUSTER +'"],' graphSpec += ' ["setMapAttribute", "defaultClusterLabel","'+ settings.DEFAULT_CLUSTER_LABEL +'"],' graphSpec += ' ["setMapAttribute", "defaultClusterShape",0],' graphSpec += ' ["setMapAttribute", "defaultClusterStyle","'+ defaultNodeStyle +'"],' graphSpec += ' ["setMapAttribute", "defaultClusterPencolor","'+ settings.DEFAULTNODE_PENCOLOR +'"],' graphSpec += ' ["setMapAttribute", "defaultClusterFillcolor","'+ settings.DEFAULT_CLUSTER_FILLCOLOR +'"],' graphSpec += ' ["setMapAttribute", "defaultTunnelFillcolor","'+ settings.TUNNELNODECOLOR +'"],' # FIXME: these two (not?) currently detected by client, have to explicitly set it on Change Message to create far end of tunnel: graphSpec += ' ["setMapAttribute", "defaultTunnelStyle","'+ defaultNodeStyle +'"],' graphSpec += ' ["setMapAttribute", "defaultTunnelLabel","'+ settings.DEFAULTLABELS[settings.TUNNEL_NODE] +'"] ' # FIXME: client isn't detecting or using this. #graphSpec += ' ["setMapAttribute", "defaultTunnelPencolor","'+ settings.TUNNEL_PENCOLOR +'"]' return(graphSpec) except: raise def convertToTextShape(shape): try: if str(shape).isnumeric(): numericShape = int(shape) if numericShape == -1: numericShape = 0 return str(nodeShapes()[numericShape]) # convert to graphviz-compatible shape description else: return(shape) except: # FIXME return shape # it was ok to begin with raise # FIXME: rename. It isn't really making nodes, but retrieving them from the db. # FIXME: can this be combined with dot4Anode? def makenodes(request, mapId, asNodeList=False, withLabel=True,forGraphvizLayout=False, excludeClusters=False): try: # FIXME: graphSpec = "" hidden_nodes = [] # FIXME: not used, maybe should be nodeList = [] # FIXME: remove nodesize from the database nodes = all_nodes_in_this_map(mapId, excludeClusters=excludeClusters) if nodes: for node in nodes: if asNodeList: nodeList.append(node) else: label=None if not node.nodetype: raise Exception("Missing nodetype in method 'makenodes'") if node.tunnelfarendmap: # FIXME: apparently not used targetmapname = Map_desc.objects.get(pk=node.tunnelfarendmap.id).mapname else: targetmapname = "" timestamp = round((node.created_date - datetime(1970,1,1,tzinfo=timezone.utc)).total_seconds()) # NOTE: depends on TZ=True in settings.py!! # NOTE: the params converted to string are for compability with the old PHP version of ideatree. Can be updated if the client (and graphviz) handles non-string values. # FIXME: generalize, using a list of applicable attributes, once it's standardized on string values. see filterNodeQuery() params = {} if node.description: params["description"] = node.description if node.targetmapname: # FIXME: used? We just looked it up above. params["targetMapName"] = node.targetmapname if node.tunnelfarendmap: params["tunnelfarendmap"] = node.tunnelfarendmap.id # FIXME fontcolor should always be determined in the canonical db, as here, so that it's consistent for exports vs. db loads. But may be now dynamically by the client to ensure readability, or both? # FIXME client js doesn't understand pencolor correctly. if node.pencolor: params["pencolor"] = str(node.pencolor) else: params["pencolor"] = "#000000" if node.nodetype: params["nodetype"] = str(node.nodetype) if node.hiddenbranch: params["hiddenbranch"] = node.hiddenbranch #if node.isbranchhead: # FIXME not found in client js #params["isbranchhead"] = node.isbranchhead if node.numcomments: params["numcomments"] = str(node.numcomments) if node.lwidth: params["lwidth"] = node.lwidth if node.url: params["url"] = node.url # FIXME "openNewWin":node.openNewWin, if node.style: # FIXME: fix client to understand just "style" params["nodestyle"] = node.style if node.fillcolor: params["fillcolor"] = node.fillcolor if withLabel: # FIXME: what's the purpose of this? params["label"]=node.label if node.shape: params["shape"] = node.shape #params["shape"] = convertToTextShape(node.shape) if forGraphvizLayout: # FIXME: hacky. needed because client thinks in 'size' of nodes, while graphviz thinks in width/height if node.size: size=node.size.split(',') params["width"] = size[0].strip(whitespace) params["height"] = size[1].strip(whitespace) else: # to be returned to client side if node.size: params["size"] = node.size #//"tunnelOutlet":tunnelOutlet); # FIXME: leave as object until the last moment before sending to client, then use ast or json.dumps to convert # FIXME: inconsistent naming is fixed here. params["ownerID"] = node.owner.id #currentUsername = request.session['username'] if request.session.get('username') else None # FIXME lookup from ownerID params["owner"] = nodeOwner params["pos"] = str(node.xpos) + "," + str(node.ypos) params["timestamp"] = timestamp if not forGraphvizLayout: graphSpec += json.dumps(makeCreateNodeChangeMessageJSON(node.id, params, forGraphvizLayout)) + "," if(node.clusterid != 0): # FIXME: for consistency, make client detect null rather than zero, then change db default graphSpec += "[\"putNodeInCluster\"," + str(node.id) +"," + str(node.clusterid) + "]," else: nodeList.append(makeCreateNodeChangeMessageJSON(node.id, params, forGraphvizLayout)) if not forGraphvizLayout and not asNodeList: return(myutils.rStrChop(graphSpec, ',')) # FIXME: check memory usage else: return(nodeList) except: raise makenodes.alters_data = True #/* Build edges between nodes. # # Queries the vectors table to find them. # # @author Ron Newman ron.newman@gmail.com #*/ # NOTE: has no concept of clusters def edgeit(mapId, endpointsAndEdgeCostsOnly=False, forGraphvizLayout=False, positiveEdgeCostsOnly=None): try: graphSpec = "," edgeList=[] edges = Edge.objects.filter(status=settings.EDGE_ACTIVE, ofmap__id=mapId) for edge in edges: edgeargs={"origin":edge.origin.id, "target":edge.target.id, "owner":edge.owner.id} edgeargs["label"] = costToLabel(edge.cost, edge.label) if (positiveEdgeCostsOnly and (edge.cost < 0)): # The dijstra algorithm requires positive or zero values. raise Warning("For this function, edge weights must be zero or greater. Cost found:" + str(edge.cost)) # This cost is independent of the edge weight in graphviz source files. edgeargs["cost"] = edge.cost # No if conditional. Could be replacing an earlier value. if edge.hiddenbranch: edgeargs["hiddenbranch"] = edge.hiddenbranch if edge.penwidth: edgeargs["penwidth"] = edge.penwidth if edge.color: edgeargs["color"] = edge.color if edge.draw: edgeargs["_draw_"] = edge.draw if endpointsAndEdgeCostsOnly: edgeList.append( (edgeargs["origin"], edgeargs["target"], edgeargs["cost"]) ) # return a triple: origin, target, cost elif forGraphvizLayout: edgeList.append(makeCreateEdgeChangeMessageJSON(edge.id, edgeargs, forGraphvizLayout)) else: graphSpec += json.dumps(makeCreateEdgeChangeMessageJSON(edge.id, edgeargs)) + "," # FIXME: output a little to a file at a time, to save memory. # FIXME: test when no edges if not forGraphvizLayout and not endpointsAndEdgeCostsOnly: return(myutils.rStrChop(graphSpec, ',')) # FIXME: check memory usage else: return(edgeList) except: raise edgeit.alters_data = True @ensure_csrf_cookie @login_required() def createNewEntityProvisional(request, forUser=None): try: try: mapId = currently_opened_map(request) except Warning as err: # Since it's a system error since it's within createProvisional (no UI), though it's a warning elsewhere, elevate this exception to a system error. raise Exception("System error:" + str(err)) # FIXME: set a default of None if not in POST numNodes = int(request.POST["provisionalIdsRequested[node]"]) # FIXME: set a default of None if not in POST numEdges = int(request.POST["provisionalIdsRequested[edge]"]) numProvisionalIdsRequested = {'node':numNodes, 'edge':numEdges } provisionals = makeProvisionals(request, mapId,forUser,numProvisionalIdsRequested) return HttpResponse(json.dumps(provisionals)) except Exception as err: return HttpResponse(json.dumps({'error':str(err)}), status=406) createNewEntityProvisional.alters_data = True @transaction.atomic() def makeProvisionals(request, mapId,forUser,numProvisionalIdsRequested): # FIXME test for invalid mapId try: if not forUser: #FIXME: do we have to set the user at all so soon, separated from activation of the provisional? userID = int(request.session["_auth_user_id"]) forUser = User.objects.get(pk=userID) ids={} ofmap = Map_desc.objects.get(pk=mapId) for entityType,num in numProvisionalIdsRequested.items(): if int(num) > 0: ids.update({entityType:[]}) for x in range(int(num)): # IMPORTANT: this HAS to return the provisional id as a string. Otherwise a bug in the client will cause new nodes to throw errors # when the mouse enters the corona. See Graph.js::createEdge, the line containing 'typeof string'. FIXME? ids[entityType].append(str(makeProvisional(ofmap, forUser, entityType))) return ids except: raise makeProvisionals.alters_data = True # A single provisional @transaction.atomic() def makeProvisional(ofmap, forUser, entityType): try: if (entityType=="node"): return(allocateNodeRecord(ofmap, forUser)) elif(entityType=="edge"): return(allocateEdgeRecord(ofmap, forUser)) else: raise Exception("Missing entity type.") except: raise def allocateNodeRecord(ofmap, thisUser): try: provisionalNode = Node.objects.create(owner=thisUser, nodetype=settings.PROVISIONAL, ofmap=ofmap) return(provisionalNode.id) except: raise def allocateEdgeRecord(ofmap, thisUser): try: # FIXME: node and edge aren't orthogonal. In node, 'provisional' is a type, in edge, it's a status. provisionalEdge = Edge.objects.create(owner=thisUser, status=settings.EDGE_PROVISIONAL, ofmap=ofmap) return(provisionalEdge.id) except: raise # FIXME: can't handle when a change message response is expected. # FIXME: test places where non-keyword calls are made! # FIXME URGENT: it's assumed throughout that status=600 will result in a popup. That's not true. # FIXME: can't be used everywhere. executeServerCommands() in the client, for instance, requires an 'error' attribute, not 'result' or 'message'. def makeResponse(status=400, body=None, content_type = 'application/json'): try: # FIXME: add error logging bodyDict={} if status==200: bodyDict["result"] = "S" if body: #bodyDict.update(body) bodyDict["message"] = body elif status==600: status=200 bodyDict["result"] = "E" if body: bodyDict["message"] = body else: bodyDict["originalStatus"] = status # preserve the original because we're about to change it bodyDict["originalMessage"] = http.client.responses[status] + ", " + body status=200 # Our own convention: the location of the error is put in the # error message, since by the time we propagate here we don't know the location. bodyDict["result"] = "E" bodyDict["message"] = "" # putting a message here would result in a popup on the client bodyDict["Expires"] = "Mon, 26 Jul 1997 05:00:00 GMT" # Date in the past bodyDict["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0" # HTTP/1.1 bodyDict["Pragma"] = "no-cache" # HTTP/1.0 # NOTE the content_type here (defaults to 'application/json' and the dataType in the originating Ajax call. See docs for jQuery ajax 'dataType'. return HttpResponse(json.dumps(bodyDict), status=200, content_type="application/json", charset="UTF-8") #return HttpResponse(json.dumps(bodyDict)) except: raise def sendToFirebase(request, chgMTrans, mapId, time): # NOTE:createEntityProvisional returns a value directly to calling client, not going through this process to route through Firebase try: if not mapId: raise Warning("sendToFirebase: Missing mapId. Change message not sent to map sharers, but may have been saved to database.") # FIXME: unset chgMTrans when not used, # NOTE: this assumes the CSRFToken will be in the top level, where it can be removed. If not, the token # won't be found and will be inadvertently broadcast to clients. # FIXME: how will csrftoken be used in change messages in context of django? #if chgMTrans["CSRFToken"]: #unlink(chgMTrans["CSRFToken"]) # FIXME move json_encode out of calls and to here chgMTrans["time"] = time # print("comingIN:" + chgMTrans) # print("\ncanonicalTime:" + canonicalTime) # print("\nsending to Firebase:" + json.dumps(chgMTrans)) if not time and not fromFile: raise Exception("sendToFirebase: Canonical time stamp missing. Change message not sent to Firebase.") if settings.USE_FIREBASE: # NOTE: This server's connection to Firebase is instantiated in def push_value() the first time it is called. push_value("changeMessages", str(mapId), chgMTrans) except Exception: raise sendToFirebase.alters_data = True # Encapsulates knowledge of change message format in one place. # FIXME: use standard json for change messages rather than a unique format that has to be parsed. class changeMsgParse(): try: def cmd(self): try: chCommand = self.changeMessage[0] if not chCommand.isalpha(): raise Exception("Illegal characters in change message command.") return chCommand except: raise def itemId(self): try: if self.changeMessage[1]: cm = self.changeMessage[1] if isinstance(cm,str): if cm.lower() == "null": # FIXME non-pythonic to check, but client often sends 'null'. return None if not cm.isalnum(): raise Exception("Change Message itemId must be numbers or letters.") return(cm) cm = int(cm) # raises exception if can't be converted return cm except: raise def attrs(self): try: if len(self.changeMessage) < 3: return {"_value":None} myattrs = self.changeMessage[2] if myattrs == 0: # putNodeInCluster can send zero as a cluster number return {"_value": 0} elif myattrs == "null": # putNodeInCluster can send zero as a cluster number return {"_value": ""} elif not myattrs: return {"_value":None} elif not isinstance(myattrs,dict): # FIXME: merge all the inconsistent change message formats into one. myattrs = {} # convert to dict if len(self.changeMessage) == 3: if isinstance(self.changeMessage[1],str): myattrs[self.changeMessage[1]] = self.changeMessage[2] elif isinstance(self.changeMessage[1],int): myattrs["_value"] = self.changeMessage[2] elif len(self.changeMessage) == 4: if isinstance(self.changeMessage[2],str): myattrs[self.changeMessage[2]] = self.changeMessage[3] #validate for key,val in myattrs.items(): if not re.match('[a-zA-Z_]',key): raise Exception("Change Message attribute KEY must be letters or underscore only: " + key) if isinstance(val,int): dummy = int(val) # simple validation elif isinstance(val,str): if val and not re.match(settings.STRING_SAFETY_REGEX,val): # FIXME: works for international? raise ValueError("Invalid characters in VALUE: " + val) return myattrs raise Exception("Change message parser does not understand the format.") except TypeError as err: raise Exception(err) except ValueError as err: raise Warning(err) raise except: raise def __init__(self,changeMessage): try: self.changeMessage = changeMessage except: raise except: raise @ensure_csrf_cookie @login_required() def changeMessageHandler(request): def _createNode(request, ownerID, nodeId, mapId, nodeAttrs): # FIXME: are both nodeAttrs and request needed? attributes are in request. # This handles a) canonical db saving and b) possible creation of change message(s) for creating/moving the far end of tunnels. try: form = ValidateNodeForm(nodeAttrs) if not form.is_valid(): raise Exception(form.errors) #fieldList = [f.name for f in Node._meta.get_fields()] # FIXME: kludgy. Get the list from the form itself, as above # FIXME: only the fields explicitly input by the user are cleaned. Assumes the others aren't tampered with during transmission from client. for name in ['label','description','url']: # these must correspond to the fields parameter in the form if nodeAttrs.get(name): nodeAttrs[name] = form.cleaned_data[name] # FIXME: there are more attributes saved in the DB that the change message protocol doesn't handle. # NOTE: Label is currently expected to be empty, and is subsequently set with setNodeAttribute(). FIXME: still true? # This will change when createNode takes over more of what setNodeAttribute does. # Client has been given a default label to use on map load. # FIXME: some of these args are sent but then re-sent using setNodeAttribute() # mapId can be the current map, or if part of a tunnel change message, the target map of the tunnel. # mapId needs to be explictly set because if missing from a tunnel CM, the default map would be the wrong one. nodetype = nodeAttrs.get("nodetype") # Convert shape to numeric value if not nodeAttrs.get("shape"): nodeAttrs["shape"] = nodeShapes('square') elif isinstance(nodeAttrs["shape"], str): # FIXME: hack to adjust for client sending string shapes shapeSentByClient = nodeAttrs.get("shape") nodeAttrs["shape"] = nodeShapes(shapeSentByClient) mapObj = Map_desc.objects.get(pk=mapId) customCheck(request, mapObj) # Make a change message for the node that's the far end of tunnel and append it to the changes received from client. # It will be created in the db on the next pass. """ if nodetype==settings.TUNNEL_NODE and not nodeAttrs.get("isTunnelFarEnd"): tunnelfarendmapId = nodeAttrs.get("tunnelfarendmap") try: have_map_access(request,tunnelfarendmapId) # check again because importing a map with tunnels doesn't have UI checks newTargetNodeCM = createTunnelTargetCM(request, userID, mapId, nodeId, nodeAttrs) if newTargetNodeCM: newT = changeMsgParse(newTargetNodeCM) targetNodeId = newT.itemId() nodeAttrs["tunnelfarendnode"] = targetNodeId _putInTransQueue(newT.attrs()["mapId"], newTargetNodeCM) except: # do nothing, just don't allow the far end to be created, because imports may contain tunnels that should be visible but go nowhere. pass """ if nodeAttrs.get("creationDate"): # FIXME: discrepancy in naming nodeAttrs["timestamp"] = nodeAttrs.get("creationDate") if nodeAttrs.get("style"): # FIXME: discrepancy in naming nodeAttrs["nodestyle"] = nodeAttrs.get("style") nodeAttrs["ownerID"] = ownerID # Special case for path begin/end: a separate table must be updated. if nodetype==settings.PATH_BEGIN_NODE: if MapPathEndpoints.objects.filter(ofmap_id=mapId, pathBeginNode_id__isnull=False).exists(): raise Warning("Sorry, there can be only one PATH ORIGIN per "+settings.WHAT_A_GRAPH_IS_CALLED+".") MapPathEndpoints.objects.update_or_create(ofmap_id=mapId, defaults={'pathBeginNode_id':nodeId}) elif nodetype==settings.PATH_END_NODE: if MapPathEndpoints.objects.filter(ofmap_id=mapId, pathEndNode_id__isnull=False).exists(): raise Warning("Sorry, there can be only one PATH END per "+settings.WHAT_A_GRAPH_IS_CALLED+".") MapPathEndpoints.objects.update_or_create(ofmap_id=mapId, defaults={'pathEndNode_id':nodeId}) # FIXME: creationDate differs from the timestamp on the transaction as a whole # FIXME URGENT: tunnels are removed during getImport, so don't appear here. # Not checking permissions for tunnels because importing a map may contain tunnels to maps you don't have access to, # and kicking out the whole import is a pain for the user. Notifying them which tunnel - one at a time - is the culprit # is hard code-wise and not much better in user experience. So we allow tunnels which link to maps they don't have # access to and rely on checks when they actually try to click that tunnel, blocking them if they don't have access to the target map. if nodetype !=settings.TUNNEL_NODE: checkPermissions(request, userID,"createNode", mapId, nodeId) attrsForDB = deepcopy(nodeAttrs) storeNodeToDB(request, mapId, ownerID, nodeId, attrsForDB, 'createNode') # This just to match what the PHP site sends to the client. Not sure if it's used. FIXME uname = User.objects.get(pk=ownerID).username nodeAttrs["owner"] = uname # NOTE: Signals can't be used for this because we never actually create a node, only update provisional nodes. Map_desc.objects.filter(pk=mapId).update(nodesleft=F('nodesleft')-1) except ValidationError as err: raise Exception(err) except: raise # FIXME: can we have a dynamic _popFromTransQueue? def _putInTransQueue(mapId, changeMsg): # Modeling from the original transaction, we append change message for something else we thought of doing, which also needs to be broadcast to listeners. try: tempTransaction = deepcopy(transTemplate) tempTransaction["mapId"] = mapId # Add a change message back into the queue, so that it can both be executed and also will ultimately sent to the real-time database # for broadcast to clients listening ON THE TARGET MAP. tempTransaction["changes"].append(changeMsg) transactions.append(tempTransaction) except: raise def _putNodeInCluster(mapId, nodeId, clusterId): try: # int() serves as validation: Node.objects.filter(pk=int(nodeId), ofmap_id=int(mapId)).update(clusterid=int(clusterId)) except: raise """ def _delete_other_end_of_tunnel(nodeId): try: tunnelFarEndMapId = lookup_node_info(nodeId, "tunnelfarendmap") tunnelFarEndNodeId = lookup_node_info(nodeId, "tunnelfarendnode") # Make a change message that deletes the far end, but doesn't follow it back to this end: if tunnelFarEndMapId and tunnelFarEndNodeId: # Excel imports, for example, can have tunnels that have no far end. CM = makeDeleteAbandonedTunnelNodeChangeMessage(tunnelFarEndNodeId, tunnelFarEndMapId) _putInTransQueue(tunnelFarEndMapId, CM) except: raise """ def _removeNode(request, nodeId, mapId, attrs=False): try: # It's important that nodetype is accurate (if a tunnel the other ends must be deleted). Though it should be passed in by javascript, here's a lookup to double-check. nodetype=lookup_node_info(nodeId, "nodetype") nodeowner=lookup_node_info(nodeId, "owner") mapId =lookup_node_info(nodeId, "ofmap") mapObj = Map_desc.objects.get(pk=mapId) # FIXME: standardize on map objects or ids # Prepare to delete any edges to or from it. edges = Edge.objects.filter(ofmap=mapId, status=settings.EDGE_ACTIVE).filter(Q(origin=nodeId) | Q(target=nodeId)) for edge in edges: CM = makeRemoveEdgeChangeMessage(edge.id, mapId) _putInTransQueue(mapId, CM) if nodetype==settings.VOTABLE_NODE: Vote.objects.filter(node_id=nodeId).update(status=settings.VOTABLE_DELETED) # Try all the most important parts. # Get all the vectors associated with this node because we have to delete them, too. # NOTE: client looks up the attached edges and sends separate messages to delete them. # FIXME: this isn't necessary if we use constraints to cascade deletes at the DB level. However, now we're not really deleting, # but marking for later deletion by cron job. This way is easier to reconstruct if client does an Undo? NodeComment.objects.filter(node=nodeId).update(status=settings.NODE_COMMENT_DELETED) # NOTE: the client sends separate removeEdge change messages as needed when a node with edges is deleted. So this isn't really needed. # FIXME: but write a test for orphan edges in case the client misses one. Node.objects.filter(pk=nodeId, ofmap=mapId).update(status=settings.NODE_DELETED) # NOTE: Signals can't be used for this because we never create nodes, only update provisional ones: # NOTE: relies on transaction rollback in order to be accurate in the case of a DB error below this line. Map_desc.objects.filter(pk=mapId).update(nodesleft=F('nodesleft')+1) # NOTE: client handles clusters: delete_cluster_if_empty(cluster) # Special case for path begin/end: a separate table must also be updated. if nodetype==settings.PATH_BEGIN_NODE: MapPathEndpoints.objects.filter(ofmap_id=mapId).update(pathBeginNode_id=None) elif nodetype==settings.PATH_END_NODE: MapPathEndpoints.objects.filter(ofmap_id=mapId).update(pathEndNode_id=None) if nodetype in settings.PATH_ENDPOINT_NODE_TYPES: # if the last path begin/end in this map, delete the record. MapPathEndpoints.objects.filter(ofmap_id=mapId, pathBeginNode=None, pathEndNode=None).delete() if nodetype==settings.VOTABLE_NODE: Vote.objects.filter(node=nodeId).update(status=settings.VOTABLE_DELETED) #eraseOutputFiles(ofMap); except ObjectDoesNotExist as err: raise Exception(err) except: raise _removeNode.alters_data = True def _setNodeAttribute(request, userID, nodeId, mapId, attrName, attrValue): # FIXME both request and attrValue needed? try: #form = ValidateNodeForm(request) #if not form.is_valid(): #raise Exception(form.errors) #fieldList = [f.name for f in Node._meta.get_fields()] #FIXME: do this once and stash somewhere # FIXME: kludgy # corresponds to the fields parameter in the form #for name in ['label','description','url']: # these correspond to the fields parameter in the form #if kwargs.get(name): #kwargs['label'] = form.cleaned_data['label'] attrData = { attrName: str(attrValue) } if attrName == 'tunnel': # FIXME: forces disregard of obsolete client change message: return None if attrName == "tunnelfarendmap": # FIXME ever used? A new tunnel sends this parameter within createNode(). requestedTargetMap = attrValue have_map_access(request,tunnelfarendmapId) # check again because importing a map with tunnels doesn't have UI checks checkPermissions(request, userID,"setNodeAttribute", mapId, nodeId) storeNodeToDB(request, mapId, userID, nodeId, attrData, 'setNodeAttribute') except: raise _setNodeAttribute.alters_data = True def _createEdge(request, userID, edgeID, mapId, kwargs): # FIXME: separate kwargs from request needed? try: form = ValidateEdgeForm(kwargs) if not form.is_valid(): raise Exception(form.errors) #fieldList = [f.name for f in Node._meta.get_fields()] #FIXME: do this once and stash somewhere # FIXME: kludgy # corresponds to the fields parameter in the form if kwargs.get("label"): kwargs['label'] = form.cleaned_data['label'] # NOTE: if kwargs in the future contains edge cost, add a test to tests.py for no costs on path endpoints. See current check in setEdgeAttributes(). # FIXME: permission didn't exist for createEdge until called from unit test. Is this function every called in # actual operation? # FIXME: kludgy check for both int and str IDs originId = targetId = None #if not isinstance(kwargs["origin"], int): # originId = int(kwargs["origin"]) if kwargs.get("origin") and isinstance(kwargs["origin"], str) and kwargs["origin"].isdigit() else None #if not isinstance(kwargs["target"], int): # targetId = int(kwargs["target"]) if kwargs.get("target") and isinstance(kwargs["target"], str) and kwargs["origin"].isdigit() else None originId = int(kwargs["origin"]) targetId = int(kwargs["target"]) # Double check for node existence. if not Node.objects.filter(pk=originId, ofmap_id=mapId, status=settings.NODE_ACTIVE).exists(): raise Warning("Origin node does not exist (maybe someone else deleted it?). ID:" + str(originId)) if not Node.objects.filter(pk=targetId, ofmap_id=mapId, status=settings.NODE_ACTIVE).exists(): raise Warning("Target node does not exist (maybe someone else deleted it?). ID:" + str(targetId)) #else: # labelOfOrigin = kwargs["origin"] # labelOfTarget = kwargs["target"] # # FIXME: make more liberal? Compile? # # FIXME: use a form to do this cleaning automatically and less restrictively # cleanedOriginLabel = re.match('[^a-zA-Z0-9_]', labelOfOrigin) # Get rid of non-alphanumeric characters except for underscore # cleanedTargetLabel = re.match('[^a-zA-Z0-9_]', labelOfTarget) # Get rid of non-alphanumeric characters except for underscore # if not cleanedOriginLabel: # raise Warning(labelOfOrigin + " contains character(s) that are not allowed in Ideatree.") # if not cleanedTargetLabel: # raise Warning(labelOfTarget + " contains character(s) that are not allowed in Ideatree.") # # NOTE: limitation: Ideatreeallows multiple nodes with the same label, but kumu.io imports only give 'From' and 'To' in terms of label, not ID. # # Check for node existence in the case of a map import or an API call, which may rely on node labels rather than internal knowledge of IDs. # labelOfOrigin = cleanedOriginLabel.group() # labelOfTarget = cleanedTargetLabel.group() # targetNode = Node.objects.filter(ofmap_id=mapId, label=labelOfTarget, status=settings.NODE_ACTIVE) # originNode = Node.objects.filter(ofmap_id=mapId, label=labelOfOrigin, status=settings.NODE_ACTIVE) # if not originNode.exists(): # raise Exception("Can't find an existing edge origin node, label:" + str(labelOfOrigin)) # if not targetNode.exists(): # raise Exception("Can't find an existing edge target node, label:" + str(labelOfTarget)) # targetId = targetNode.id # originId = originNode.id checkPermissions(request, userID,"createEdge", mapId, targetId, edgeID) # FIXME: use named parameters # Special cases for edges to/from path endpoint nodes. targetnodetype = str(Node.objects.filter(pk=targetId, status=settings.NODE_ACTIVE).values_list('nodetype',flat=True)[0]) originnodetype = str(Node.objects.filter(pk=originId, status=settings.NODE_ACTIVE).values_list('nodetype',flat=True)[0]) if targetnodetype==settings.PATH_BEGIN_NODE: raise Warning("You can't connect TO a path-beginning node, only FROM it. Sorry.") elif originnodetype==settings.PATH_END_NODE: raise Warning("You can't connect FROM a path-ending node, only TO it. Sorry.") # FIXME: remove if not used: #if originnodetype==settings.PATH_BEGIN_NODE: #existingOutEdges = Edge.objects.filter(origin=originId, status=settings.NODE_ACTIVE).count() #if existingOutEdges >= 1: #raise Warning("Only one connection is allowed from a Begin Path node.") # if targetnodetype==settings.PATH_END_NODE: # existingInEdges = Edge.objects.filter(target=targetId, status=settings.NODE_ACTIVE).count() # if existingInEdges >= 1: # raise Warning("Only one connection is allowed to an End Path node.") # NOTE: Edge(s) status can be 'Provisional', or if created in the process of an Undo, 'Deleted', so don't filter by status. edgeargs=deepcopy(kwargs) if '_draw_' in edgeargs: # Don't use get() because _draw_ can also be null. edgeargs['draw'] = edgeargs.pop('_draw_') # Django won't let field names end with an underscore, but the Graphviz attribute is _draw_. Edge.objects.filter(id=edgeID).update(ofmap=mapId, status=settings.EDGE_ACTIVE, **edgeargs) except FieldDoesNotExist as err: raise Exception(err) except: raise _createEdge.alters_data = True def _removeEdge(request, userID, edgeID, mapId): try: checkPermissions(request, userID,"removeEdge", mapId, None, edgeID) Edge.objects.filter(ofmap=mapId).filter(id=edgeID).update(status=settings.EDGE_DELETED) except Edge.DoesNotExist as err: raise Exception(err) except: raise _removeEdge.alters_data = True def _setEdgeAttribute(request, ownerID, edgeId, mapId, kwargs): # FIXME: separate kwargs from request needed? try: form = ValidateEdgeForm(kwargs) if not form.is_valid(): raise Exception(form.errors) #fieldList = [f.name for f in Node._meta.get_fields()] #FIXME: do this once and stash somewhere # FIXME: kludgy # corresponds to the fields parameter in the form if kwargs.get("label"): kwargs['label'] = form.cleaned_data['label'] # FIXME write test of every permutation of permission: own/don't own, allowed/not allowed. checkPermissions(request, ownerID,"setEdgeAttribute", mapId, None, edgeId) # FIXME: repeated in _createEdge if '_draw_' in kwargs: # Don't use kwargs.get() because _draw_ can legitimately be null. kwargs['draw'] = kwargs.pop('_draw_') # Django won't let field names end with an underscore, but the Graphviz attribute is _draw_. kwargs["id"] = edgeId kwargs["ofmap"] = mapId kwargs["owner"] = ownerID if kwargs.get("label"): # FIXME: this trim isn't getting propagated out to CM to send through firebase. kwargs["label"] = myutils.addslashes(kwargs["label"].strip(whitespace)) # trim empty space FIXME: do it this way elsewhere if kwargs.get("cost"): newLabel = kwargs.get("label") newLabel = costToLabel(kwargs["cost"], newLabel) # setEdgeAttribute handles one thing at a time, so make a separate change messages and push it on the queue for ChangeMessageHandler to do. CM = makeSetEdgeAttributeChangeMessage(edgeId, "label", newLabel) _putInTransQueue(mapId, CM) # FIXME: validate NewNodeForm this way, checking id, ofmap, and owner. #If we allow edge costs to/from Path endpoint nodes it throws off shortest path calculations, so don't allow it. edgeEndpoints = list(Edge.objects.filter(pk=edgeId, status=settings.EDGE_ACTIVE).values_list('origin','target')) originId = edgeEndpoints[0][0] targetId = edgeEndpoints[0][1] originnodetype = str(Node.objects.filter(pk=originId, status=settings.NODE_ACTIVE).values_list('nodetype',flat=True)[0]) targetnodetype = str(Node.objects.filter(pk=targetId, status=settings.NODE_ACTIVE).values_list('nodetype',flat=True)[0]) showNoEdgeCostWarning=False if originnodetype==settings.PATH_BEGIN_NODE or targetnodetype==settings.PATH_END_NODE: if kwargs.get("cost"): showNoEdgeCostWarning=True kwargs["cost"]=settings.DEFAULTEDGE_COST if not kwargs.get("cost"): # If cost changed to empty string in the UI kwargs["cost"]=settings.DEFAULTEDGE_COST form = AlterEdgeForm(kwargs) if form.is_valid(): Edge.objects.filter(id=edgeId, ofmap=mapId, status=settings.EDGE_ACTIVE).update(**kwargs) if showNoEdgeCostWarning: raise Warning("Sorry, connections to/from " + settings.PATH_ORIGIN_LABEL + "
or " + settings.PATH_END_LABEL + " cannot be assigned weights.") return(True) else: raise Exception(form.errors) except: raise _setEdgeAttribute.alters_data = True def _setMapAttribute(request, ownerID, kwargs): form = ValidateMapForm(kwargs) if not form.is_valid(): raise Exception(form.errors) try: # FIXME: coordinate DPI with client by sending it on map load as a default. if kwargs.get("defaultNodeWidth"): kwargs["defaultNodeWidth"] = round(kwargs["defaultNodeWidth"]/settings.DOTS_PER_INCH,2) if kwargs.get("defaultNodeHeight"): kwargs["defaultNodeHeight"] = round(kwargs["defaultNodeHeight"]/settings.DOTS_PER_INCH,2) checkPermissions(request, ownerID,"setMapAttribute", request.session["mapId"]) form = AlterMapForm(kwargs) if form.is_valid(): Map_desc.objects.filter(id=request.session["mapId"]).update(**kwargs) else: raise Exception(form.errors) except: raise _setMapAttribute.alters_data = True def _doHandler(request, userID, cmTrans): try: mapId = cmTrans['mapId'] for change in cmTrans['changes']: cmP = changeMsgParse(change) chgAtr = cmP.attrs() itemId = cmP.itemId() cmd = cmP.cmd() #if itemId == 7: if itemId: if cmd=='createNode': _createNode(request, userID, itemId, mapId, chgAtr) elif cmd=='putNodeInCluster': _putNodeInCluster(mapId, itemId, list(chgAtr.values())[0] ) elif cmd=='setNodeAttribute': attrName = list(chgAtr)[0] # extract key/value from a dictionary of a single entry attrValue = list(chgAtr.values())[0] _setNodeAttribute(request, userID, itemId, mapId, attrName, attrValue) #if chgAtr.get("shape"): # FIXME #chgAtr["shape"] = str(chgAtr["shape"]) elif cmd=='removeNode': _removeNode(request, itemId, mapId, chgAtr) elif cmd=='createEdge': _createEdge(request, userID, itemId, mapId, chgAtr) elif cmd=='removeEdge': mapId = chgAtr.get("mapId") if (chgAtr and chgAtr.get("mapId")) else mapId _removeEdge(request, userID, itemId, mapId) elif cmd=='setEdgeAttribute': # FIXME: ugly hack to compensate: after a graphviz layout, the first move of a connected node sends _draw_:null from the # client. This MUST go to firebase as '_draw_':'', or the edge won't reload in the new position. if "_draw_" in chgAtr and not chgAtr['_draw_']: chgAtr['_draw_'] = "" change[3] = "" _setEdgeAttribute(request, userID, itemId, mapId, chgAtr) elif cmd=='setMapAttribute': _setMapAttribute(request, userID, chgAtr) except: raise def _sendCM2Firebase(request, transact, transactionId): # Processing above has successfully saved to canonical db at this point, so it's safe # to attempt broadcast of the same change messages to clients, via firebase, who will update their local maps. try: # Set before sending to firebase in case firebase send fails. request.session["lastCompletedTransactionId"] = transactionId # FIXME Remove attributes meaningful only to the canonical db. Or just let the clients ignore them. mapId = transact.get("mapId") if settings.USE_FIREBASE: sendToFirebase(request, transact, mapId, request.session["tableLockTime"]) return None except: # FIXME: is this true? This applies in the rare condition that the ideatree server is available, but not firebase; # and even if so, does the ideatree server firebase instance have a retry mechanism? # Hacky translation to an understandable prompt #if "Unable to find the server" in str(err): #err = "Due to a telecommunications interruption viewers of this map
\ #will not immediately see the change you just made.
\ #
\ #However, they will automatically be updated after we have
\ #re-established communication." #raise Warning(err) # FIXME: test #else: raise # MAIN of changeMessageHandler try: # FIXME: rename 'originalClientTransaction', 'originalTransaction' # ** # Why POST is used instead of PUT: http://jcalcote.wordpress.com/2008/10/16/put-or-post-the-rest-of-the-story/ # See also: http://www.markbaker.ca/2002/08/HowContainersWork/ # ** originalClientTransaction=json.loads(request.POST["chngMsgTrans"]) # NOTE mapId from the *initial* change message is not used, but taken from the more secure session. mapId = currently_opened_map(request) originalClientTransaction["mapId"] = mapId userID = int(request.session["_auth_user_id"]) if 'readOnly' in request.session and request.session["readOnly"]: raise Exception("This action not allowed for non-editing users.") transactions = [] transactions.append(originalClientTransaction) transTemplate = deepcopy(originalClientTransaction) del transTemplate["changes"] transTemplate["changes"] = [] # FIXME can there ever be more than one transaction? If not, why the loop below? transactionId = originalClientTransaction['transactionId'] # FIXME: how to validate this? # Detect and return without action a client retry of a change message that completed after the client timeout, # where the client thinks it failed when it didn't. if transactionId == request.session.get("lastCompletedTransactionId"): return makeResponse(status=200, body=None) # NOTE: any updates to tables that Change Messages or API calls look at should be done in a db transaction. # The client also has its own transaction with rollback, and we need to do similar to keep in sync. with transaction.atomic(): # FIXME: test rollback. # Get a fresh timestamp, which should be set at time of processing the first Change Message in the transaction received and as close to # the first db change as possible. request.session["tableLockTime"]=myutils.millisNow() if not request.session.get("lastCompletedChangeMessageTime") or (request.session["lastCompletedChangeMessageTime"] <= request.session["tableLockTime"]): request.session["lastCompletedChangeMessageTime"] = request.session["tableLockTime"] else: # FIXME: this exception fails silently. Desired behavior? raise Exception("Change message received that's older than others that have been completed."); # NOTE: transactions["changes"] can be dynamically added to within _doHandler() for trans in transactions: _doHandler(request,userID, trans) # FIXME: should firebase error (or just going offline) invalidate the whole transaction as it does? How about a separate loop? _sendCM2Firebase(request, trans, transactionId) return makeResponse(status=200, body = {"transactionId":transactionId } ) # FIXME: test these exception handlers with client-level test (Selenium or Django's alternative) except Warning as err: # FIXME if the message is shorter than 'Message', the header of the popup, the popup looks strange. Also, 'Message' should be centered in its header. Client code. # How messages are returned if a change message transaction is expected: err = "
"+ str(err) + "
" # NOTE: the 'E' result tells the client not to display whatever was created, but status OK causes the error message to be displayed in a popup. return HttpResponse(json.dumps({"transactionId":transactionId, "result":"E", "message":err}), status=200) except PermissionDenied as err: err = "
"+ str(err) + "
" return HttpResponse(json.dumps({"transactionId":transactionId, "result":"E", "message":err}), status=200) except ObjectDoesNotExist as err: # FIXME: could plain DoesNotExist be used instead, which could identify the model it came from? status=406 body=str(err) return makeResponse(status, body) except Exception as err: return makeResponse(status=406, body=str(err)) changeMessageHandler.alters_data = True @transaction.atomic() def db_to_chgMsgFile(request, mapId, graph_fp): try: #setFontSizes() FIXME # set as close as possible to table lock time. # FIXME: the api/controller hierarchy has a class method to do this # FIXME: DIY for all sets of canonicalTime canonicalTime= str(myutils.millisNow()) transID = str(random.randrange(1000000)) # ******************************** preliminaries ************************************************************* data = makeTransactionHeaderJSON(mapId, transID, canonicalTime) data += makeGraphDefaults(request,mapId) + "," # ******************************** nodes & edges ************************************************************* data += str(makenodes(request,mapId)) data += edgeit(mapId) data = myutils.rStrChop(data, ',') # In case there are no nodes. data += "]}" graph_fp.write(data) return(True) except: raise #** # # Background: The graph files, in the form of .json formatted change message, are kept in a # separate directory for each map named by # the map ID (57, 133, etc). The map ID is the row number of the Map_desc table. # We assume only ONE graph file exists in the output directory. # We look up the filenames, because when starting a new session, the random value filename # is unknown. # # @author Ron Newman #**/ def map_meta(request,mapId,provisionalIdsRequested,userID): try: graph = {} graph["provisionalIds"] = provisionalIdsRequested graph["secondsToStaleProvisional"] = settings.PROVISIONAL_SECONDS_TO_STALE # client javascript needs something to be here graph["debug_output"]= "" # FIXME: used? # FIXME: actually used by client? graph["prompt"] = "

You can create a " + settings.WHAT_A_NODE_IS_CALLED + " by dragging one of the icons at the upper right to this area.

" if 'readOnly' in request.session: # we set read only status per user, in their session, rather than by map graph["readOnly"]=1 # FIXME: use constants or a filepath class. # FIXME: ROideatree no longer used. if(not os.path.isfile("ROideatree/"+ str(mapId) + "/public.json")): graph["image"]='{"error":"Map has not been published"}' else: graph["image"]="ROideatree/"+ str(mapId) +"public.json" else: graph["readOnly"] = 0 if(('grfFormat' in request.session) and request.session["grfFormat"]==settings.OUTLINEFORMAT): graph["image"]=getHtmlMap(get_relativeOutfilePath(mapId)) else: graph["image"]= get_outfileURL(mapId) graph["mapOwnerName"]= request.session["map_owner_name"] graph["mapUsers"] = all_map_members(mapId, idsOnly=False) graph["mapDescription"]=request.session["mapDescription"] graph["mapname"]=request.session["mapname"] # FIXME graph["mapPublishInfo"] = getPublishedDateDescription(mapId) # in the case of a bookmarklet click, automatically create a popup requiring # submit (for security), populated with data from the bookmarklet. # Commented out until security can be thought about. #if(("remoteURI" in request.session) and request.session["remoteURI"]): #remoteURI=request.session["remoteURI"] #if("remoteTitle" in request.session) and request.session["remoteTitle"]) #remoteTitle = request.session["remoteTitle"][:settings.NODE_LABEL_LENGTH-1] # don't need these anymore #request.session["remoteURI"]=None #request.session["remoteTitle"]=None # FIXME: clickDocketParams still valid on the client? #graph["clickDocketParams"] = "fromBookmarklet=1&nodetype=".settings.ORIGINAL_NODE."&label="+remoteTitle+"url="+remoteURI graph["result"] = 'S' return HttpResponse(json.dumps(graph)) except Exception as err: # FIXME: json? return HttpResponse("System error:" + str(err), status=406) # Step 2 of opening a map. # What the client calls to get a map. Along with provisional nodes & edges, it returns the "image" in the form of a url # that the client will load. The url is to a file holding the actual map change messages for creating a map. @ensure_csrf_cookie @login_required() def mapLoader(request): try: mapId = int(request.POST["mapId"]) userID = int(request.session["_auth_user_id"]) thisUser = User.objects.get(pk=userID) # FIXME test have_map_access(request, mapId) # Check if a re-rendering of the map is needed. # FIXME # if needs_regenerate: if not 'readOnly' in request.session: updateMapSessionData(request, mapId) # Create the file to hold the change messages we've generated from database. # Many ways to check existence of file/directory, some that introduce a race condition. # See http://stackoverflow.com/questions/273192/how-to-check-if-a-directory-exists-and-create-it-if-necessary/14364249#14364249 # Why not to use flock for locking, and other things: # http://chris.improbable.org/2010/12/16/everything-you-never-wanted-to-know-about-file-locking/ # Because I want global, across-processes locking, I this is interesting: # http://stackoverflow.com/questions/186202/what-is-the-best-way-to-open-a-file-for-exclusive-access-in-python # (scroll down to "I solved it myself") # But that solution isn't widely used. So I settled on writing to a temporary file and renaming it. Renaming is atomic in Python, on both Win and Linux after Python 3.3. # http://stackoverflow.com/questions/1348026/how-do-i-create-a-file-in-python-without-overwriting-an-existing-file # and # https://docs.python.org/3/library/tempfile.html outputDir = make_absoluteOutfileDir(mapId) clearOutputFiles(outputDir) # used when this function called in the process of doing layout or export outfilePath = make_absoluteOutfilePath(mapId, settings.MAPLOAD_TEMP_FILE_SUFFIX) graph_fp = tempfile.NamedTemporaryFile("w+t", encoding="UTF-8",delete=False) # generate the map in the form of change messages db_to_chgMsgFile(request,mapId,graph_fp) # atomic, in case some client is currently reading outfile. # FIXME: stress test extensively with many concurrent clients # NOTE: must close a temporary file before moving it. Thus the 'delete=False' attribute above. graph_fp.close() # WARNING: don't use os.rename(), Windows will fail if file exists. shutil.move(graph_fp.name, outfilePath) # create the provisional objects they want to keep around in this map for node creation. # FIXME: DIY with createNewEntityProvisional() # int() is for cheap validation numNodes = int(request.POST["provisionalIdsRequested[node]"]) numEdges = int(request.POST["provisionalIdsRequested[edge]"]) provisionalIdsRequested = {'node':numNodes, 'edge':numEdges } provisionalIds = makeProvisionals(request, mapId, thisUser, provisionalIdsRequested) # for purposes of chat currentMap = Map_desc.objects.get(pk=mapId) WhosViewingMap.objects.update_or_create(user=thisUser, mapViewed=currentMap) data = map_meta(request,mapId,provisionalIds,userID) return HttpResponse(data, status=200, content_type="application/json", charset="UTF-8") else: # whatever is needed for read only embedded maps goes here pass except Warning as err: body = str(err) status = 600 return makeResponse(status, body) except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) except Exception as err: return HttpResponse("System error:" + str(err), status=406) mapLoader.alters_data = True @login_required() def chatWin(request): try: mapId = currently_opened_map(request) # FIXME: figure out how to do with with a context, so {{ whatagraphiscalled }} can be included. return HttpResponse(open(get_absoluteTemplatesDir() + 'chatWin.html','r')) except FileNotFoundError as err: raise Exception("FileNotFoundError:" + str(msg)) except Warning as err: # FIXME: test context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") # FIXME document the diff between the multiple def names that concern opening a map @ensure_csrf_cookie @login_required() @verified_email_required() def map(request): try: userId = int(request.session["_auth_user_id"]) user = User.objects.get(pk=userId) userProfile = UserProfile.objects.get(user=user) request.session["username"] = user.username readOnly = request.session.get('readOnly') # logout to clear out the session if moving from browser page logged in as read only # to a browser page logged in as regular user if readOnly: if request.GET.get('rou'): return HttpResponseRedirect('signout') # FIXME: shorten, use get() dataMobile = request.session['mobile'] if 'mobile' in request.session and request.session['mobile'] != None else None dataBrowserVersion = request.session['browserVersion'] if 'browserVersion' in request.session and request.session['browserVersion'] != None else None # make map hash survive a login. # FIXME: write a test for this # FIXME: set lifetime, etc., for cookies and session # FIXME: any security concerns here? if request.COOKIES.get('mapHash'): temp = request.COOKIES['mapHash'] if len(temp) > 1: temp = temp.split("#",1)[1] request.session['mapId'] = temp thisMapHash = "#" + temp # FIXME: Be sure to delete the cookie below, before returning # FIXME: test, because this used to be just 'if', not 'elif' elif request.session.get('mapId'): # NOTE: depends on 'APPEND_SLASH' in settings.py if readOnly: # FIXME: read-only maps have been superceded by pdf publishing thisMapHash = "#RO" + str(request.session['mapId']) + "/" else: thisMapHash = "#" + str(request.session['mapId']) + "/" else: thisMapHash = ' ' # NOTE it's a space, not blank. # FIXME: is the check for username necessary? if (request.session.get("username") and not request.session.get("is_student")): showOpenMenu = True showNewMenu = True showMapInfoMenu = True showTeamCollaborationMenus = True else: showOpenMenu = False showNewMenu = False showMapInfoMenu = False showTeamCollaborationMenus = False paidAccount = userProfile.accounttype in [settings.REGULAR_ACCT, settings.PREMIUM_ACCT] isMobile = True if 'isMobile' in request.session and request.session['isMobile'] == True else False nodeManageInstructions = tooltips.node_manage_instructions() nodeCreateInstructions = tooltips.node_create_instructions() emptyNodeTooltip = tooltips.empty_node_tooltip() emptyMapTooltip = tooltips.empty_map_tooltip() nodeTooltip = tooltips.node_tooltip_for_readonly() if readOnly else tooltips.node_tooltip() tunnelTooltip = tooltips.tunnel_tooltip() node_shapes = json.dumps(nodeShapes()) noMinify = False # FIXME: make a choices-like structure in settings.py possiblenodetypes = json.dumps({'original':settings.ORIGINAL_NODE, 'tunnel':settings.TUNNEL_NODE, 'votable':settings.VOTABLE_NODE, 'pathbeginnode':settings.PATH_BEGIN_NODE, 'pathendnode':settings.PATH_END_NODE}) username = request.session["username"] # FIXME: check length of username for long usernames. mapsettings = lookup_user_info(userId, 'mapsettings') # NOTE: these are actually user-specific, not map-specific, but this is a convenient place to load them. myrandomnum = "?__="+str(random.randrange(1000000)) context = { 'node_pin_layoutengine':settings.NODE_PINNED_LAYOUT_ENGINE, 'useFirebase':settings.USE_FIREBASE, 'currentUsername':username, 'userAuthenticated':True, 'guestUsername': settings.GUEST_USERNAME, 'what_an_edge_is_called':settings.WHAT_AN_EDGE_IS_CALLED, 'what_a_node_is_called':settings.WHAT_A_NODE_IS_CALLED, 'what_a_cluster_is_called':settings.WHAT_A_CLUSTER_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'mapsettings':mapsettings, 'noMinify':noMinify, 'thisMapHash':thisMapHash, 'cluster_label_length':settings.CLUSTER_LABEL_LENGTH, 'seconds_to_stale':settings.PROVISIONAL_SECONDS_TO_STALE, 'showOpenMenu':showOpenMenu, 'showNewMenu':showNewMenu, 'showMapInfoMenu':showMapInfoMenu, 'showTeamCollaborationMenus':showTeamCollaborationMenus, 'readOnly':readOnly, 'paidAccount':paidAccount, 'isMobile':isMobile, 'nodeManageInstructions':nodeManageInstructions, 'nodeCreateInstructions':nodeCreateInstructions, 'emptyNodeTooltip':emptyNodeTooltip, 'emptyMapTooltip':emptyMapTooltip, 'nodeTooltip':nodeTooltip, 'tunnelTooltip':tunnelTooltip, 'node_shapes':node_shapes, 'possiblenodetypes':possiblenodetypes, 'myrandomnum':myrandomnum, 'whataclusteriscalled':settings.WHAT_A_CLUSTER_IS_CALLED, 'what_an_edge_is_called':settings.WHAT_AN_EDGE_IS_CALLED, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } response = render(request,'map_page.html', context) response.delete_cookie('mapHash') return response except Exception as err: return HttpResponse("System error:" + str(err), status=406) map.alters_data = True @ensure_csrf_cookie @login_required() # @gzip_page() # FIXME # @never_cache() # FIXME def mapPullImport(request,mapId): try: # Ignore the mapId passed here and use only the mapId set in the session by mapOpen mapId = currently_opened_map(request) have_map_access(request, mapId) request.session["mapId"] = mapId # FIXME: delete temp files such as this either when not needed or by cron job when no users are logged in to this map. path = os.path.join(make_absoluteImportDir(mapId),"changeMessages.json") return HttpResponse(open(path),'r') except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) except FileNotFoundError as err: raise Exception("FileNotFoundError:" + str(err)) except Exception as err: return HttpResponse("System error:" + str(err), status=406) @ensure_csrf_cookie @login_required() # @gzip_page() # FIXME # @never_cache() # FIXME def mapPull(request,mapId): try: # FIXME: make client allow deleting a map if pull did not complete. # FIXME test # Ignore the mapId passed here and use only the mapId set in the session by mapOpen #mapId = currently_opened_map(request) # FIXME have_map_access(request, mapId) request.session["mapId"] = mapId # FIXME: delete returnMap.html #return render(request,'returnMap.html', {'relativePathToMapImage':relativePathToMapImage } ) # FIXME: shouldn't this be 'get_absoluteOutfilePath()' since mapOpen already did a make dir? # FIXME: make the output file a more descriptive name, like '.mapChngMsgs.json' # FIXME: delete temp files such as this either when not needed or by cron job when user isn't logged in. return HttpResponse(open(make_absoluteOutfilePath(mapId, settings.MAPLOAD_TEMP_FILE_SUFFIX),'r')) except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) except FileNotFoundError as err: raise Exception("FileNotFoundError:" + str(msg)) except Exception as err: return HttpResponse("System error:" + str(err), status=406) def allNodesInMap(self, mapId): try: # FIXME: does order_by make the id__in= clause afterward any more efficient? return Nodes.objects.filter(ofmap=mapId).values_list('ofmap',flat=True).order_by('id') except: raise allNodesInMap.alters_data = True def allEdgesInMap(self, mapId): try: # FIXME filter by target, too? return Edge.objects.all().filter(origin__mapId=mapId) except: raise allEdgesInMap.alters_data = True @login_required() @user_passes_test(not_guest_user) def myAccount(request): # FIXME: test: induce form field errors as a test of template layout thisUser = User.objects.get(pk=int(request.session["_auth_user_id"])) # ignore what may be sent and always use the session userProfile = UserProfile.objects.get(user=thisUser) paidAccount = userProfile.accounttype in [settings.REGULAR_ACCT, settings.PREMIUM_ACCT] context = {'paidAccount':paidAccount } try: if request.POST.get("submitted"): # We can't check for GET/POST because with Ajax everything is a POST. instance = User.objects.get(pk=thisUser.id) form = MyAccountForm(request.POST, instance=instance) context.update({'form':form}) if form.is_valid(): form.save() context.update({'prompt':'Your info has been saved.'}) return render(request, 'ideatree/myAccount.html', context) else: form = MyAccountForm({'first_name':thisUser.first_name, 'last_name':thisUser.last_name, 'email':thisUser.email}) if form.is_valid(): context.update({'form':form}) return render(request, 'ideatree/myAccount.html', context) else: raise Exception(form.errors) except IntegrityError as err: # for form.save() return HttpResponse(err, status=406) except Exception as err: return HttpResponse(err, status=406) myAccount.alters_data = True @ensure_csrf_cookie @login_required() # FIXME: deprecated. Invite/accept sequence no longer used. def myPendingInvites(request): try: # FIXME: should be in a transaction myPendingInvites = Friend.objects.filter(friend_id=int(request.session["_auth_user_id"]), status=settings.FRIEND_INVITED).count() return HttpResponse(int(myPendingInvites)) except: raise myPendingInvites.alters_data = True def checkPermissions(request, submitterID,methodName,mapId,nodeID=None,edgeID=None,ownerID=None): try: # FIXME: put in a memcache. This is doing a DB query every time a change message is received. # NOTE: Checking map access is done here rather than when save is done because this saves a round trip to the client # if access is not granted, since saving is done only on the second trip up from the client, when they send a change message. have_map_access(request, mapId) if 'readOnly' in request.session: raise Exception(settings.NOT_PERMITTED_PROMPT) # Get the ownerID from the database, not passed by them. # Don't use session["mapId"] because that forces the API to select a map before operating on it, # and if the developer forgets the permissions may be given for the wrong map. # FIXME: rather than cover all types of map objects, just require that ownerID be sent. if not ownerID: if(nodeID): # FIXME: change 'ofmap' to another name, then integrate in filter as 'ofmap__Map_desc.id=mapId' # FIXME: overkill to make sure the map exists? ofmap = Map_desc.objects.get(pk=mapId) ownerID = Node.objects.get(pk=nodeID, ofmap=ofmap).owner.id elif(edgeID): # FIXME: change 'ofmap' to another name, then integrate in filter as 'ofmap__Map_desc.id=mapId' ofmap = Map_desc.objects.get(pk=mapId) ownerID = Edge.objects.get(pk=edgeID,ofmap=ofmap).owner.id else: ownerID = Map_desc.objects.get(pk=mapId).owner.id # In case session lifetime is set to something other than SESSION_EXPIRE_AT_BROWSER_CLOSE, check that the session login hasn't timed out. if not request.session.keys(): raise Exception("Can't check permissions. Has your access been revoked? Or is it time to login again?") # Map owner has ownership permissions for everything in the map if they created it. # FIXME: use get() if("map_owner" in request.session and int(request.session["map_owner"])==int(request.session["_auth_user_id"])): iOwn = True elif(ownerID==submitterID): iOwn = True else: iOwn = False permission = ClientPermission.objects.get(action=methodName, iown=iOwn) # return the permission if not permission.permitted: if "remove" in methodName.lower(): settings.NOT_PERMITTED_PROMPT += " NOTE: in general, you must be the creator of an item to delete it." raise Warning(settings.NOT_PERMITTED_PROMPT) return(True) except ClientPermission.DoesNotExist as err: raise Warning(settings.NOT_PERMITTED_PROMPT) except Node.DoesNotExist as err: raise Exception(err) except Edge.DoesNotExist as err: raise Exception(err) except Map_desc.DoesNotExist as err: raise Exception(err) except: raise checkPermissions.alters_data = True def makePopupTitle(request, nodetype=None): try: popupTitle = "" if 'fromBookmarklet' in request.POST: popupTitle = "OK to Insert?" elif nodetype==settings.ORIGINAL_NODE: popupTitle = settings.WHAT_A_NODE_IS_CALLED.capitalize() elif nodetype==settings.TUNNEL_NODE: popupTitle = "Tunnel" elif nodetype==settings.VOTABLE_NODE: popupTitle = "Solution to be Voted Upon" return popupTitle except: raise # FIXME: unused def getMapList(request, mapId=None, excludeSelf=True): try: mapList = [] update_mymap_list(request) # expensive, but safe, to do this every time. mymaps = json.loads(request.session["accessible_maps"]) mymapsSorted = sorted(mymaps, key=lambda k: k['mapname']) wanted_keys = ['id', 'mapname'] for amap in mymapsSorted: # Get the attributes needed for a select dropdown, and also eliminate the current map from the list. if excludeSelf: mapList.append({k:amap[k] for k in wanted_keys if amap['id'] != mapId}) else: mapList.append({k:amap[k] for k in wanted_keys}) return(mapList) except: raise def getTunnelDialogContext(request,mapId): try: context = {} context["title"] = "Create a " + settings.WHAT_A_NODE_IS_CALLED + " which tunnels..." context["tunnelOriginMapname"] = get_truncated_mapname(request,mapId) return(context) except: raise @ensure_csrf_cookie # Because render(), which enforces csrftoken checking, is not the only output here. @login_required() @csrf_protect def nodeDialog(request): # HOW THIS WORKS: # 1) Client requests node form (by clickDocket() in js), for editing an existing or a provisional node. # 2) Form is sent to client . # 3) Client submits form to nodeDialog(). # 4) If form is approved, a js "execute" command is returned, inducing client to create a change message. # 5) Client sends 'createNode' and/or 'setNodeAttribute' or 'removeNode' change message back to server, handled elsewhere. # 5b) In the case of a 'createNode' message for a tunnel node to another map, or for editing a tunnel node to change the target map, # the current inlet and outlet of the tunnel are deleted, a new outlet created in the database, and # 5c) a CM of the tunnel outlet sent to clients to cause them to update their local maps. # FIXME: for repositioning tunnel only. Verify that this doesn't use 'setNodeAttribute' instead of 'createNode'. # 6) Node is created in the database. # 7) Node is sent to clients via Firebase, in the respective maps. try: context = {'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} nodetype = None mapId = currently_opened_map(request) mapObj = Map_desc.objects.get(pk=mapId) nodesleft = mapObj.nodesleft # FIXME: test when nodesleft has run out. # FIXME: didn't client used to send nodetype? #handleURL = True # FIXME # FIXME: does this function ever get called for clusters? context.update({"nodesleft":nodesleft, 'what_a_node_is_called':settings.WHAT_A_NODE_IS_CALLED }) nodetype = request.POST.get("nodetype") if request.POST.get("nodetype") else lookup_node_info(request.POST["nodeID"],"nodetype") if nodetype==settings.TUNNEL_NODE: maps4select = all_accessible_maps(request, int(request.session["_auth_user_id"]), exclude_current=True) if nodetype==settings.TUNNEL_NODE else None context["popupTitle"] = makePopupTitle(request, nodetype) # Step 3. if request.POST.get('submitted') or nodetype in settings.PATH_ENDPOINT_NODE_TYPES: if nodetype==settings.TUNNEL_NODE: form = NewTunnelNodeForm(request.POST, queryset=maps4select, label_suffix='') else: form = NewNodeForm(request.POST, label_suffix='') # NOTE: we can't require nodetype in the model because of provisional nodes, which have no nodetype, kind of like a stem cell. if form.is_valid(): customCheck(request, mapObj, forEditDisplay=True) # FIXME: this should be done in form clean if possible # As far as possible, we take values from the form, since they're validated. argsForClient = {} argsForClient.update(form.cleaned_data) if (request.POST.get("nodeID")): argsForClient["nodeID"]=int(request.POST.get("nodeID")) # the one exception that unfortunately isn't available in the form. if form.cleaned_data.get("tunnelfarendmap"): argsForClient["tunnelfarendmap"] = form.cleaned_data.get("tunnelfarendmap").id argsForClient.update({"targetMapName":get_truncated_mapname(request,form.cleaned_data["tunnelfarendmap"].id)}) # For use of the client, to name the tunnel origin node. jsToExec = issueJsExec([argsForClient]) # Tell the client to generate change messages to send back here, which we will use to # alter the database and broadcast to all other listening clients via firebase. #FIXME: what if client doesn't respond? Does client timeout take care of that? If not, figure out how to flag that with an exception. return HttpResponse(json.dumps([jsToExec])) else: # prepare to re-render the form with errors marked. context["form"] = form if form.cleaned_data["nodetype"] == settings.TUNNEL_NODE: context.update(getTunnelDialogContext(request,mapId)) return render(request, 'ideatree/tunnelDialog.html', context ) else: return render(request, 'ideatree/nodeDialog.html', context) # Client requested a form loaded with a specific node for editing or creation (using a provisional node previously sent to the client). # FIXME: use instead http://stackoverflow.com/questions/433162/can-i-access-constants-in-settings-py-from-templates-in-django/25841039#25841039 # FIXME: resolve 'id' vs. 'nodeID' vs. 'nodeId' # FIXME: put 'nodesLeft' in tunnelDialog.html # Step 1. elif request.POST.get('nodeID'): nodeId = int(request.POST.get("nodeID")) # some rudimentary validation node = Node.objects.get(pk=nodeId, ofmap=mapId) context["nodeId"] = nodeId if node.nodetype == settings.TUNNEL_NODE: context.update(getTunnelDialogContext(request,mapId)) form = NewTunnelNodeForm(queryset=maps4select, label_suffix='', instance=node) form.is_valid() # initiate validation #FIXME returns 'unknown' context["form"] = form return render(request, 'ideatree/tunnelDialog.html', context ) else: form = NewNodeForm(label_suffix='', instance=node) form.is_valid() # initiate validation context.update({'form':form, 'urlLength':settings.NODE_URLFIELD_LENGTH, 'descriptionLength':settings.NODE_DESCRIPTION_LENGTH, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED}) return render(request, 'ideatree/nodeDialog.html', context) # An empty form for tunnel was requested from the docket # Alt. Step 1 elif nodetype == settings.TUNNEL_NODE: form = NewTunnelNodeForm(request.POST, queryset=maps4select, label_suffix='') # FIXME Don't validate yet because it won't pass until data has been entered. context.update(getTunnelDialogContext(request,mapId)) context["nodetype"] = request.POST.get("nodetype") context["form"] = form return render(request, 'ideatree/tunnelDialog.html', context ) # An empty non-tunnel node creation form was requested from the docket # Alt. Step 1 elif request.POST.get("nodetype"): form = NewNodeForm(request.POST, label_suffix='') # FIXME Don't validate yet because it won't pass until data has been entered. context["form"] = form return render(request, 'ideatree/nodeDialog.html', context) else: raise Exception("Invalid request.") except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(err, status=406) def customCheck(request, mapObj, forEditDisplay=False): try: if not forEditDisplay and not mapObj.nodesleft > 0: raise Warning(settings.MAP_FULL_PROMPT) if not request.POST.get("label") and request.POST.get("url"): raise Warning("Need a label if address is provided.") # NOTE: if a label is empty it is handled in the client using map defaults sent on map load. except: raise def issueJsExec(listOfArgs, forAppend=False): try: cmds = [] jsToExec={} for argsForClient in listOfArgs: cmd = {} if argsForClient.get("action"): action = argsForClient.get("action") argsForClient.pop("action") else: action = "" #print(argsForClient) #argsForClient.pop("action") # FIXME: EditNodeForm uses the same action for both nodes and votable nodes. Better? # NOTE: these don't really matter as far the server is concerned, 'createNode' should make the differentiation since the differences # are all in the arguments. if argsForClient.get("nodeID"): # FIXME: for readability, make the action explicit in the passed arguments. action = "alterNode" elif argsForClient.get("nodetype") == settings.ORIGINAL_NODE: action = "createNode" elif argsForClient.get("nodetype") == settings.TUNNEL_NODE: action = "createTunnel" elif argsForClient.get("nodetype") == settings.VOTABLE_NODE: action = "createVotableNode" elif argsForClient.get("nodetype") == settings.PATH_BEGIN_NODE: action = "createNode" elif argsForClient.get("nodetype") == settings.PATH_END_NODE: action = "createNode" elif argsForClient.get("edgeId"): action = "setEdgeAttribute" if argsForClient.get("tunnelfarendmap"): # FIXME: naming translation. argsForClient["tunnel"] = argsForClient["tunnelfarendmap"] # Remove empty fields # FIXME: do with list comprehension cleanArgs = {} for key,val in argsForClient.items(): if val: cleanArgs[key] = val cmd["args"] = cleanArgs cmd["action"] = action cmds.append(cmd) # At this point, form submission successful, ready to echo back an 'execute' command. Client will # generate a change message that does the actual node creation by sending another request # which causes the new node to be created in the db. If successful, we report success then # and client keeps the locally created node. # initial instruction to client to start creating the tunnel origin and target. # FIXME if request.POST["nodetype"]==settings.VOTABLE_NODE: # FIXME: done by client now?: slider = new views_SliderInput($this,"voteLevel",0,"",null,null,null) # NOTE: the following attributes are combined with map defaults for sliders in db_to_chgMsgFile(). if not forAppend: jsToExec["execute"] = cmds return jsToExec return cmds except Exception as err: raise issueJsExec.alters_data = True # FIXME: use NewNodeForm to validate this data, even though it comes through a change message, not a form. # FIXME: remove callerFuncName when the client and back end are in synch # FIXME: security hole: it's theoretically possible for a client to get approval for a transaction from server, # then the client to alter node ids before sending the request back to the server. Would still be limited to within the # current map, and they'd have to guess node ids that exist in that map. def storeNodeToDB(request, mapId, ownerID, itemID, data, callerFuncName): # NOTE: See note in checkPermissions as to why haveMapAccess is done there rather than here. try: #FIXME: shouldn't this be "ownerID"? data["owner"] = ownerID # Override anything sent in a change message, for security. if data.get("url"): data["url"] = data["url"].replace(" ","_") # replace spaces with underlines in URLs # All this just to limit characters of 'size' field. if data.get("size"): data["size"] = data["size"].strip(whitespace) size = data["size"].split(',') # additional split is just to remove potential white space before or after. It returns a list, so [0] gets the value out. # FIXME: necessary? dimW = '{:.2f}'.format(float(size[0])) # convert exponential float to one long number dimH = '{:.2f}'.format(float(size[1])) #dimW = round(float(size[0].strip(whitespace)), 2) #dimH = round(float(size[1].strip(whitespace)), 2) data["size"] = str(dimW) + ',' + str(dimH) # FIXME: client sends an improper string shape for createNode, but the right numerical shape for setNodeAttribute if data.get("xpos"): data["xpos"] = '{:.3f}'.format(float(data["xpos"])) if data.get("ypos"): data["ypos"] = '{:.3f}'.format(float(data["ypos"])) # NOTE: data["created_date"] doesn't need translation because it's already in a format that matches the db. # convert pos to x,y floats. # FIXME db actually doesn't have a 'pos' attribute. Be consistent. # FIXME: floats are 64 bits each in Postgresl, so not a space saver. Can Graphviz understand if we leave it as pos? if data.get("pos"): xypos = data["pos"].split(',') # additional split is just to remove potential white space before or after. It returns a list, so [0] gets the value out. # FIXME: necessary? data["xpos"] = '{:.3f}'.format(float(xypos[0])) data["ypos"] = '{:.3f}'.format(float(xypos[1])) data["pos"] = None # Assumed faster than pop(). Will be filtered out by filterNodeQuery(). # FIXME: standardize on '#' or not if data.get("fillcolor"): data["fillcolor"] = data["fillcolor"].replace("#","") # FIXME: standardize on '#' or not if data.get("fontcolor"): data["fontcolor"] = data["fontcolor"].replace("#","") if data.get("label"): # When graphfiz 'fixedsize' parameter is set to true, to reduce Graphviz "label too long" warnings. data["label"] = myutils.truncateStr(data["label"], settings.NODE_LABEL_LENGTH) # Alternatively: #if len(data["label"]) > settings.NODE_LABEL_LENGTH: #raise Warning(str(settings.WHAT_A_NODE_IS_CALLED).capitalize() + " label length is limited to " + str(settings.NODE_LABEL_LENGTH) + " characters.
It's suggested that you use the "+ str(settings.WHAT_A_NODE_IS_CALLED) + "'s Description to give longer information.") data["label"] = data["label"].replace('"','') # NOTE: client correctly escapes double quotes, so firebase gets a clean escaped version. kwargs = filterNodeQuery(data) if 'id' in kwargs: # Shouldn't have gotten this far, but check just in case, because it causes subtle bugs. # We will retrieve the row using the id ('itemID') of a row previously defined as provisional, # and update that row to some non-provisional nodetype. kwargs.pop('id') # NOTE: we're using get() instead of update() because: a) update(), using a filter, is more dangerous, can affect a whole slew of rows, and b) update doesn't # call the model.__save__() method, and we need to do stuff there. # See: https://stackoverflow.com/questions/33809060/django-update-doesnt-call-override-save nodeToUpdate = Node.objects.get(pk=itemID, ofmap=mapId) for key,val in kwargs.items(): if key=="tunnelfarendmap": # FIXME: hacky if val==mapId: raise Exception("A tunnel can't lead to itself, i.e. the origin graph.") val = Map_desc.objects.get(pk=val) if key=="owner": # FIXME: hacky val = User.objects.get(pk=val) #print("STORE NODE: SETATTR:" + " key:" + str(key) + " val:" + str(val)) setattr(nodeToUpdate, key, val) setattr(nodeToUpdate, 'status', settings.NODE_ACTIVE) # NOTE: this node update can theoretically come via either createNode or setNodeAttribute, so we put it here rather than within either of those methods. try: nodeToUpdate.save() except ValidationError as err: raise Exception(err) # Why is this here? FIXME reInstateComments(varList["ID"]) except: raise storeNodeToDB.alters_data = True # Remove attributes that aren't actually in the model. def filterNodeQuery(data): # FIXME: once client and server are in sync, this can be removed. try: kwargs = {} fieldList = [f.name for f in Node._meta.get_fields()] #FIXME: do this once and stash somewhere for field in data: if field.lower() in fieldList: kwargs[field.lower()] = data[field] else: pass #print("Wasteage: createNode function: Attribute sent by client for storing that's not supported by the database: " + field) return kwargs except: raise def broadcastNumCommentsToClients(request, node, mapId, canonicalTime): # NOTE: side effect: updates Node's numcomments field. try: out = [] out.append("setNodeAttribute") out.append(node.id) out.append("numcomments") numcomments = NodeComment.objects.filter(node=node,status=settings.NODE_COMMENT_ACTIVE).count() Node.objects.filter(pk=node.id, ofmap=mapId).update(numcomments=numcomments) out.append(numcomments) data = makeTransactionHeaderJSON(mapId, None, canonicalTime) data += json.dumps(out) data += "]}" sendToFirebase(request, json.loads(data), mapId, canonicalTime) except: raise @login_required() @csrf_protect def crudNodeComments(request): # IMPORTANT: once and for all, understand that hidden fields are necessary: # https://stackoverflow.com/questions/4673985/how-to-update-an-object-from-edit-form-in-django # Creating a node each time from id rather than passing around a node instance in templates because Django can't send # a node (ForeignKey) as a hidden var. # Do NOT pass the user or mapId as hidden fields. These provide a cross-check of validity, and taken from the session. # See also: # https://stackoverflow.com/questions/43959790/how-do-i-include-the-id-field-in-a-django-form try: # write test: # FIXME: accepts empty empty (though correctly doesn't email it even if the checkbox is checked) thisUser = User.objects.get(pk=int(request.session["_auth_user_id"])) # ignore what may be sent and always use the session mapId = currently_opened_map(request) ofmap = Map_desc.objects.get(pk=mapId) nodeID = int(request.POST.get('nodeID')) # FIXME: standardize spelling on 'nodeId' node = Node.objects.get(pk=nodeID, ofmap=mapId) nodeLabel = node.label regarding = (nodeLabel[:75] + '..') if len(nodeLabel) > 75 else nodeLabel if request.POST.get('commentId'): commentId = int(request.POST["commentId"]) if 'submitNewComment' in request.POST: instance = NodeComment.objects.create(node=node, user=thisUser) form = NodeCommentForm(request.POST, instance=instance) # combine with the new text of the comment if form.is_valid(): canonicalTime = myutils.millisNow() # Just before db update. form.save() broadcastNumCommentsToClients(request, node, mapId, canonicalTime) if request.POST.get('notifySharers'): notify(request, showUI=False, regarding=nodeLabel) elif 'submitToDelete' in request.POST: comment = request.POST["comment"] context = {'nodeLabel':nodeLabel, 'nodeID':nodeID, 'commentId':commentId, 'comment':comment } return render(request,'ideatree/nodeCommentDeleteAreYouSure.html', context) elif 'submitToUpdate' in request.POST or 'reallyDelete' in request.POST: instance = NodeComment.objects.get(pk=commentId, node=node, user=thisUser) form = NodeCommentForm(request.POST, instance=instance) # combine with the new text of the comment if 'submitToUpdate' in request.POST: if form.is_valid(): form.save() if request.POST.get('notifySharers'): notify(request, showUI=False, regarding=nodeLabel) elif 'reallyDelete' in request.POST: if form.is_valid(): canonicalTime = myutils.millisNow() # Just before db update. NodeComment.objects.filter(pk=commentId,node=node,user=thisUser).update(status=settings.NODE_COMMENT_DELETED) broadcastNumCommentsToClients(request, node, mapId, canonicalTime) # Display or re-display all the comments for this node. emptyform = NodeCommentForm() prevNodeComments = [] forms = [] prevNodeComments += NodeComment.objects.filter(node=node, status=settings.NODE_COMMENT_ACTIVE).order_by('-date') # descending (- sign) if prevNodeComments: for comment in prevNodeComments: forms.append(NodeCommentForm(instance=comment)) # FIXME: DRY with the same query in notify() sharers = Mapmember.objects.filter(ofmap=mapId, status=settings.MAPMEMBERSHIP_ACTIVE).exclude(member_id=thisUser.id) context = {'sharers':sharers} # Passed through html form to (optionally) notify sharers by email. context.update({'nodeLabel':nodeLabel, 'forms':forms, 'form':emptyform, 'nodeID':nodeID, 'thisUserId':thisUser.id, 'whatnodeiscalled':settings.WHAT_A_NODE_IS_CALLED}) return render(request,'ideatree/nodeCommentEdit.html', context) except IntegrityError as err: # for form.save() return HttpResponse(err, status=406) except Exception as err: return HttpResponse(str(err), status=406) crudNodeComments.alters_data = True # FIXME: layoutengine, mapId, format, and orientation are needed in various places. Rather than calling this every time or passing them around, # maybe a global object or vars in namespace. def getOrientation(request, mapId): return request.POST['orientation'] if request.POST.get('orientation') else lookup_map_info(mapId,"orientation") def getLayoutengine(request): return request.POST['layoutengine'] if request.POST.get('layoutengine') else settings.DEFAULT_LAYOUT_ENGINE # FIXME: use variable number of arguments def makeLayoutFile(request, haveGraphvizSrc=False, nodeListOfDicts=None, edgeListOfDicts=None): try: mapId = currently_opened_map(request) orientation = getOrientation(request, mapId) layoutengine = getLayoutengine(request) set_map_property(mapId,fieldname="layoutengine", fieldval=layoutengine) userId = int(request.session["_auth_user_id"]) # FIXME: use this lots of places: # NOTE: another possible engine that may be sent is fdp, which is used instead of neato because neato doesn't support clusters. checkPermissions(request, userId,"prettify", mapId) dotInputFile = make_absoluteImportDir(mapId) # FIXME: more accurately, 'get_absoluteImportDir' dotInputFile += "/uploaded.dot" # By naming convention. layoutResultFile = make_absoluteOutfilePath(mapId,settings.GRAPHVIZ_OUTPUT_FILE_SUFFIX) if not haveGraphvizSrc: Graph2dotFile(mapId, orientation, layoutengine, dotInputFile, nodeListOfDicts, edgeListOfDicts) # Even when we already have dot source, do a layout to get it into easily-parsed json format. VERY expensive. Would not be needed if there were a # dot2json converter: graphvizLayout(mapId, dotInputFile, layoutResultFile, layoutengine) Map_desc.objects.filter(id=mapId).update(orientation=orientation, layoutengine=layoutengine) return(layoutResultFile) except Warning as err: raise except Exception as err: return HttpResponse(json.dump(str(err)), status=406) @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() def prettify(request, provisionalNodeIds=0, provisionalEdgeIds=0, fromGraphvizSrc=False, format=None): try: layoutResultFile = makeLayoutFile(request) layoutengine = getLayoutengine(request) mapId = currently_opened_map(request) if layoutengine in [settings.CIRCULAR_LAYOUT_ENGINE, settings.RADIAL_LAYOUT_ENGINE]: numClusters = Node.objects.filter(nodetype=settings.CLUSTER, ofmap_id=mapId, status=settings.NODE_ACTIVE).count() if numClusters > 0: body = "Circular or Radial layouts aren't free to
reposition certain nodes.\n\nConsequently, Circular and Radial layouts are
not enabled when "+ settings.WHAT_A_CLUSTER_IS_CALLED +"s are present.

To enable such layouts to work, remove "+ settings.WHAT_A_CLUSTER_IS_CALLED +"s by
right-clicking each one and selecting 'Un"+ settings.WHAT_A_CLUSTER_IS_CALLED +" Entities'." return makeResponse(600, body) # Since we did a layout, we have the map in the form of a json file, which needs to be converted to change messages for the client. changes = parseLayout(request, layoutResultFile) # FIXME: A kludge: we have to convert a list of change messages to a dict of dicts for this special case of what initiatePull("prettify") # expects on the client. When pulling from file, initiatePull expects a list. out = {} for change in changes: if len(change)==4: params = {change[2] : change[3]} else: params = change[2] ch = {str(change[0])+","+str(change[1]):params} out.update(ch) out.update({"result" : "S"}) #ch = {"setMapAttribute,52":{"orientation":"TB"}} out.update(ch) return HttpResponse(json.dumps(out), status=200, content_type="application/json", charset="UTF-8") except Warning as err: return JsonResponse({'message':str(err)}, status=406) except Exception as err: return HttpResponse(str(err), status=406) prettify.alters_data = True # FIXME: use variable number of arguments def parseLayout(request, layoutFile, countOnly=False, provisionalNodeIds=None, provisionalEdgeIds=None, makeItems=False): # FIXME: look at parsing xdot (1/10 the size of json) using http://pydoc.net/Godot/0.1.1/godot.xdot_parser/ import simplejson as json try: mapId = currently_opened_map(request) orientation = getOrientation(request, mapId) layoutengine = getLayoutengine(request) edgesOnly = True if layoutengine == settings.NODE_PINNED_LAYOUT_ENGINE else False # FIXME: is 'usingfdp' needed now that I have the flipY.gvpr program? usingfdp = True if layoutengine in ['fdp'] else False currently_opened_map(request) with open(layoutFile, 'r', encoding='utf-8') as jsonMap: # explicit encoding because lack of it has thrown errors when deploying from test to production database. mapdata=jsonMap.read() mapObj = json.loads(mapdata) itemCount = {} changes=[] reassignedNodeIds = () clusterContents={} if 'objects' in mapObj and not edgesOnly: for node in mapObj['objects']: # Clusters and Nodes # if importing from dot, _gvid (starts with 0,1,2) is used. If layout from db, then the record id (large number) is used. nodeId = node.get('id') or node.get('ID') or node.get("_gvid") # in this order if countOnly: itemCount['nodes'] = itemCount.get('nodes', 0) + 1 continue # Process if a cluster. # FIXME: Simpler to use the graphviz way of specifying clusters in json output, rather than encoding it in the ID as IdeaTree does it. But how to specify the invisible node in each cluster? if node["name"].startswith("cluster") or node["name"].startswith(settings.CLUSTER_INVISIBLE_NODE_FOR_EDGE_PLACEMENT): if usingfdp: # fdp doesn't support clusters continue nodeId = node.get("ID") or node.get("_gvid") # ID when set by Ideatree database or _gvid when from an import. if "nodes" in node: clusterContents.update({nodeId:node["nodes"]}) # These values are always the _gvid values (integers starting with 0). node["nodetype"]=settings.CLUSTER # NOTE: isdigit() below is used to filter out the alphanumeric temp nodes we put in clusters for purposes of graphviz #if (isinstance(nodeId,str) and nodeId.isdigit()): #nodeId = int(nodeId) if nodeId == None: raise Exception("parseLayout: Could not find a node ID.") if provisionalNodeIds: # such as when coming from import node["nodeID"]= provisionalNodeIds.pop() # The new Ideatree-assigned ID reassignedNodeIds += ((str(nodeId),node["nodeID"]),) else: node["nodeID"]=nodeId # such as when coming from database if "nodetype" in node and node["nodetype"]==settings.CLUSTER: if node["_gvid"] in clusterContents: clusterContents[node["nodeID"]] = clusterContents.pop(node["_gvid"]) else: for cluster in clusterContents: # point the node contained to its parent cluster if node["_gvid"] in clusterContents[cluster]: node["clusterid"]=cluster if 'label' in node and node['label'] != "\\N": label = node['label'] elif 'name' in node: label = node['name'] else: label = None if label: label = myutils.truncateStr(label, settings.NODE_LABEL_LENGTH) convBB = convertGraphvizUnits('bb',node['bb'], usingfdp) if 'bb' in node else None # bb = bounding box. NOTE: Assumes Graphviz only puts in 'bb' for clusters. if convBB: height = convBB['height'] width = convBB['width'] pos = convBB['pos'] else: pos = convertGraphvizUnits('pos',node['pos'], usingfdp) if 'pos' in node else None height = node['height'] if 'height' in node else None width = node['width'] if 'width' in node else None shape = node.get("shape") shape = nodeShapes(shape) if shape else None # convert to numeric shape that client understands color = node.get("fillcolor") or node.get("color") color = color[1:] if color and color.startswith("#") else color if color: try: # Is it hexadecimal? int(color, 16) except ValueError: try: color = Colors.objects.get(colorname=color).hexvalue # Alternative: # import matplotlib # print(matplotlib.colors.cnames["blue"]) # --> u'#0000FF' except Colors.DoesNotExist: color = None # Use client's default. if makeItems: params = {"label":str(label), "shape":shape} if "nodetype" in node: params.update({"nodetype":str(node["nodetype"])}) if shape: params.update({"shape":str(shape)}) if color: params.update({"fillcolor":str(color)}) #if 'clusterid' in node: #if len(reassignedNodeIds): #foundCluster=False #for ndx in range(len(reassignedNodeIds)): #if not foundCluster: #pair = reassignedNodeIds[ndx] #if pair[0]==str(node["clusterid"]): #node["clusterid"] = pair[1] #foundCluster=True #if not foundCluster: #raise Exception("Parent cluster not found for node, original id:"+ str(nodeId)) changes.append(makeCreateNodeChangeMessageJSON(node["nodeID"], params)) #size = str(width) + "," + str(height) #NOTE: all 3, height, width, width have to be in message or client will complain. if not pos: pos = "0,0" changes.append(makeSetNodeAttributeChangeMessage(node["nodeID"], {"height": str(height), "pos":str(pos), "width":str(width)})) #if 'clusterid' in node and foundCluster: if 'clusterid' in node: changes.append(makePutNodeInClusterChangeMessage(node["nodeID"], node["clusterid"])) if 'edges' in mapObj: for edge in mapObj['edges']: if countOnly: itemCount['edges'] = itemCount.get('edges', 0) + 1 continue if provisionalEdgeIds: edge["edgeID"] = provisionalEdgeIds.pop() else: edge["edgeID"] = edge.get("ID") or edge.get("id") or edge.get("_gvid") # in this order if 'origin' in edge: origin = edge['origin'] elif 'tail' in edge: origin = edge['tail'] else: origin = None if 'target' in edge: target = edge['target'] elif 'head' in edge: target = edge['head'] else: target = None originFound = targetFound = False if len(reassignedNodeIds): for ndx in range(len(reassignedNodeIds)): pair = reassignedNodeIds[ndx] #print("checking as origin:" + str(pair[0]) + " to match:" + str(origin)) if not originFound and pair[0]==str(origin): # FIXME: sometimes origin is int and sometimes str, depending on import source format. originFound=True edge["origin"] = pair[1] #print("checking as target:" + str(pair[0]) + " to match:" + str(target)) if not targetFound and pair[0]==str(target): # FIXME: sometimes target is int and sometimes str, depending on import source format. edge["target"] = pair[1] targetFound=True if not originFound and not targetFound: raise Exception("Endpoint not found for edgeID:" + str(edge["edgeID"])) #cost = edge['cost'] if 'cost' in edge else None pincolor = edge['pincolor'] if 'pincolor' in edge else None # The client doesn't currently use pos for edges, but we use it to add last segment of the edge. pos = edge['pos'] if 'pos' in edge else None _draw_ = edge['_draw_'] if '_draw_' in edge else None lastPoint = None if pos: lastPoint = pos.split(' ').pop(0).replace('e,','').strip() lastPoint = [float(f) for f in lastPoint.split(',')] if _draw_: # Make last point of _draw_ match what first entry in 'pos' (which is the last point in the line) says, # since the client isn't accurately accounting for the arrowhead size FIXME if lastPoint: points = _draw_[len(_draw_)-1].get("points") # FIXME: a kludge to account for varying _draw_ lengths. Find reason for variation. if points: points.pop() points.append(lastPoint) else: raise Exception("Unable to parse _draw_ points.") _draw_ = convertGraphvizUnits("_draw_", _draw_, usingfdp) # FIXME: what about style, width, LABEL? if makeItems and origin and target: changes.append(makeCreateEdgeChangeMessageJSON(edge["edgeID"], {"origin":edge["origin"],"target":edge["target"]})) # _draw_ is useful (for prettify? re-route edges?), but messes up the client if sent at edge creation time. # FIXME: make sure _draw_ is used elsewhere, else delete the code above. if _draw_: # Because of the stupid way that the client expects a dict of dicts (meaning keys are unique), subsequent change messages # added to the dict for a given edge will overwrite previous changes. So we can only send one AlterEdgeChangeMessage per edge. changes.append(makeAlterEdgeChangeMessage(edge["edgeID"], "_draw_", str(_draw_))) #if cost: #changes.append(makeAlterEdgeChangeMessage(edge["edgeID"], "cost", str(cost))) #if pincolor: #changes.append(makeAlterEdgeChangeMessage(edge["edgeID"], "pincolor", str(pincolor))) if countOnly: return(itemCount) else: return(changes) except FileNotFoundError as err: raise Exception("FileNotFoundError:" + str(msg)) except ValueError as err: # FIXME: test this #msg = err.msg + " at:" + err.pos # FIXME: threw an error once msg = err raise Exception(msg) except Warning as err: raise except Exception as err: raise def convertGraphvizUnits(attr, val, usingfdp=False): try: if not val: return("") if(attr=="bb"): # "bb" = bounding box, calculated by graphviz vals=val.split(",") vals0 = float(vals[0]) vals1 = float(vals[1]) vals2 = float(vals[2]) vals3 = float(vals[3]) sizeX=vals2 - vals0 sizeY=vals3 - vals1 bb = {} # points to inches, two decimal places bb['width'] = "%.2f" % (sizeX / settings.DOTS_PER_INCH) bb['height'] = "%.2f" % (sizeY / settings.DOTS_PER_INCH) #bb['width'] = "%.2f" % (sizeX / 72) #bb['height'] = "%.2f" % (sizeY / 72) # find center of bounding box xpos = "%.2f" % (vals2-(sizeX / 2)) # -1 to convert from graphviz (lower left) origin to our (upper left) origin. # Values from DB are flipped before graphviz handles them. Here we flip back. # Finally, fdb improperly flips when nodes are pinned to positions, so no need to flip again. if usingfdp: ypos = "%.2f" % (-1 * (vals3-(sizeY / 2))) else: ypos = "%.2f" % (vals3-(sizeY / 2)) pos = str(xpos)+","+str(ypos) bb['pos'] = pos return(bb) elif(attr=="pos"): vals = val.split(",") xpos = round(float(vals[0]),2) # flip to convert from graphviz (lower left) origin to our (upper left) origin # Values from DB are flipped before graphviz handles them. Here we flip back. #FIXME why? # Finally, if using fdb, it improperly flips when nodes are pinned to positions, so no need to flip again. if usingfdp: ypos = round((-1 * float(vals[1])),2) else: ypos = round(float(vals[1]),2) return(str(xpos) + "," + str(ypos)) elif(attr=="_draw_"): drawCmd = "" for attr in val: op = attr.get("op") if op.lower() == "s": drawCmd += " " + op drawCmd += " " + str(len(attr.get("style"))) drawCmd += " -" + attr.get("style") elif op.lower() == "c": drawCmd += " " + op drawCmd += " " + str(len(attr.get("color"))) drawCmd += " -" + attr.get("color") if op.lower() == "b": drawCmd += " " + op.upper() # FIXME: the client is too picky, regex in Translation.js::pathDownTranslate() accepts only 'B' drawCmd += " " + str(len(attr.get("points"))) for point in attr.get("points"): # fdb improperly flips when nodes are pinned to positions, so no need to flip again. if usingfdp: drawCmd += " " + str(point[0]) + " " + str(-1 * point[1]) # flip y position else: drawCmd += " " + str(point[0]) + " " + str(point[1]) # don't flip anything # example command: drawCmd = " S 5 -solid c 7 -#000000 B " + str(numPoints) + points return(drawCmd) except: raise def graphvizLayout(mapId, dotInputFile, resultFile, layoutengine=settings.DEFAULT_LAYOUT_ENGINE ): try: resultFileType = os.path.splitext(resultFile)[1][1:] flags = ' -n -s'+ str(settings.DOTS_PER_INCH) if layoutengine in ['neato','fdp'] else "" if resultFileType==settings.GRAPHVIZ_OUTPUT_FILE_SUFFIX: layoutcommand = layoutengine + flags + " -T"+resultFileType +" -o "+resultFile+ " " + dotInputFile # FIXME: document why shlex was used instead of plain split() args = shlex.split(layoutcommand) output = run(args,capture_output=True, text=True) if output.returncode != 0: errStr = output.stderr.decode("utf-8") raise Warning(errStr) elif resultFileType==settings.PUBLISH_FILE_SUFFIX: # NOTE: PDF, while requiring an additional chained execution component, has the advantage over SVG that it can be zoomed within the page. # It has the disadvantage that it can be downloaded, so a security risk if the author changes their mind and unpublishes the page. try: # First, create an empty file so permissions can be set: # Best discussion: https://therenegadecoder.com/code/how-to-check-if-a-file-exists-in-python/ with open(resultFile, "r"): pass except FileNotFoundError: fp=open(resultFile, "wt") fp.close() # NOTE: shell=True only usable securely because no user-input is part of the call. # ps2pdf is a utility included in graphviz, but it requires ghostscript. Alternatively, it's also included in many Linux distributions. # NOTE: at this point, whoami is wsgiuser. We create permissions so layout can write the pdf result: myutils.doWSGIPermissions(resultFile, mask=stat.S_IRWXU | stat.S_IRWXG ) # To flip Y values around the X-axis: # fdp (or neato) with pin=true is used in the first step in order to preserve interactive editing changes that have been made via the client. # NOTE: neato doesn't support clusters. That's why we use fdp here. # FIXME: change to run() out = check_output(['fdp -s'+ str(settings.DOTS_PER_INCH) + ' -n -Tdot ' + dotInputFile + ' | gvpr -c -f '+ settings.GVPR_FLIPY_PATH +' | fdp -s72 -Tps2 | ps2pdf - ' + resultFile], shell=True) # FIXME: check return status of the above line # Change the permissions of the pdf file to allow the webserver to display it: myutils.doWSGIPermissions(resultFile, mask=stat.S_IRWXU | stat.S_IRWXG ) else: raise Exception ("Unknown output file format:" + resultFileType) except FileNotFoundError as err: raise Exception("FileNotFoundError:" + str(msg)) except CalledProcessError as err: raise Exception(err) except OSError as err: raise Exception("OSError:" + str(err)) except Exception: raise def jsonToJScommands(request,layoutResultFile,provisionalNodeIdList,provisionalEdgeIdList): try: # FIXME: look at parsing xdot (1/10 the size of json) using http://pydoc.net/Godot/0.1.1/godot.xdot_parser/ # FIXME: coalesce with duplicate code in parseLayout() import simplejson as json nodeList = edgeList = [] currently_opened_map(request) with open(layoutResultFile, 'r', encoding='utf-8') as jsonMap: # explicit encoding because lack of it has thrown errors when deploying from test to production database. mapdata=jsonMap.read() mapObj = json.loads(mapdata) importedNodes = mapObj['edges'] importedEdges= mapObj['objects'] _makeEdgesFromImport(request, importedNodes, importedEdges, provisionalEdgeIdList) _makeNodesFromImport(request, importedNodes, provisionalNodeIdList) except: raise def imports(request): try: return render(request, "ideatree/imports.html") except: raise #@user_passes_test(not_guest_user) # FIXME @login_required() def download(request): # https://stackoverflow.com/questions/1156246/having-django-serve-downloadable-files # https://stackoverflow.com/questions/48949022/django-filewrapper-memory-error-serving-big-files-how-to-stream try: # FIXME: write test with map names that have non-alphanum chars and with utf-8 chars #from django.http import FileResponse from django.http import StreamingHttpResponse from wsgiref.util import FileWrapper #django >1.8 mapId = currently_opened_map(request) # Derive the filename to be downloaded so it doesn't have to passed. # FIXME URGENT: is request.POST cleaned by Django? No. See https://stackoverflow.com/questions/33829305/django-how-to-validate-post-parameters, format = request.POST.get("format").lower() if format in [settings.GRAPHVIZ_GV.lower(), settings.GRAPHVIZ_DOT.lower()]: the_file = get_absoluteExportPath(mapId,format) # Reason for using octet-stream: https://forum.graphviz.org/t/mime-type-for-downloading-dot-files/877 content_type = "\"application/octet-stream\"" elif format==settings.EXCEL_FILE_SUFFIX: the_file = get_absoluteExportPath(mapId,settings.EXCEL_FILE_SUFFIX) # Just in case you don't want to use 'application/octet-stream' below. content_type = "\"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\"" elif format==settings.PUBLISH_FILE_SUFFIX: the_file = get_absoluteExportPath(mapId,settings.PUBLISH_FILE_SUFFIX) # Just in case you don't want to use 'application/octet-stream' below. content_type = "\"application/pdf\"" else: raise Exception("File format not found in request.") filename = Map_desc.objects.get(pk=mapId).mapname filename = (filename[:25]) if len(filename) > 25 else filename filename = filename.replace(" ", "_") filename = re.sub('[^a-zA-Z0-9_]', '', filename) # Get rid of non-alphanumeric characters except for underscore filename += "." + format chunk_size = 8192 # FIXME useful?: #response = FileResponse(the_file, content_type="application/pdf") # https://docs.djangoproject.com/en/2.0/ref/request-response/#django.http.StreamingHttpResponse # FIXME: catch the FileNotFound exception here response = StreamingHttpResponse( FileWrapper(open(the_file, 'rb'), chunk_size), content_type="application/octet-stream") response['Content-Length'] = os.path.getsize(the_file) response['Content-Disposition'] = 'attachment; filename="%s"' % filename return response """ # Alternate download methods: # FIXME remove these if the above works on production server # 1) if True: # https://docs.djangoproject.com/en/1.11/howto/outputting-csv/#streaming-large-csv-files response = HttpResponse(content_type='application/force-download') response['Content-Disposition'] = 'attachment; filename="'+downloadFileAs+'"' return response # 2) if True: # for local debugging using django runserver from django.views.static import serve return serve(request, os.path.basename(filepath), os.path.dirname(filepath)) # 3) else: # for production using Apache2 from django.utils.encoding import smart_str response = HttpResponse(content_type='application/force-download') response['Content-Disposition'] = 'attachment; filename=%s' % smart_str(downloadFileAs) # Why not to use x-sendfile mod on apache: https://serverfault.com/questions/879130/is-mod-xsendfile-deprecated #response['X-Sendfile'] = smart_str(path_to_file) # It's usually a good idea to set the 'Content-Length' header too. # You can also set any other required headers: Cache-Control, etc. return response """ except Exception: raise @login_required() def makeExcelFile(request, mapId, workbook): try: # Make a worksheet for nodes worksheet = workbook.add_worksheet('nodes') cell_integer_format = workbook.add_format() cell_integer_format.set_num_format(1) cell_float_format = workbook.add_format() cell_float_format.set_num_format('0000000.000') # NOTE: a) ofmap is not set because it requires a Map_desc object, not just an id. # FIXME: use lowercase 'id' columnHeaders = ( ['ID','label', 'nodetype','fillcolor','group','clusterid','created_date', 'owner', 'targetmapname', 'tunnelfarendmap','status','xpos','ypos','size','lwidth','shape','style','pencolor','fontcolor','numcomments','url','description'],) row = 0 col = 0 # Write column headers of node worksheet for header in columnHeaders[0]: worksheet.write_string(row, col, header) col += 1 row = 1 col = 0 # Write node data to node worksheet # FIXME: automate the cell type detection nodeData = Node.objects.filter(ofmap_id=mapId, status=settings.NODE_ACTIVE).exclude(nodetype = settings.PROVISIONAL) for node in nodeData: # row num worksheet.write_number(row,col, node.id, cell_integer_format) col += 1 #label = models.CharField() if node.label: worksheet.write_string(row,col, node.label) else: worksheet.write_string(row,col, settings.EMPTY_NODE_LABEL_PLACEHOLDER) col += 1 # nodetype = models.CharField(max_length=1) if node.nodetype == settings.ORIGINAL_NODE: worksheet.write_string(row,col, 'Node') elif node.nodetype == settings.TUNNEL_NODE: worksheet.write_string(row,col, 'Tunnel') elif node.nodetype == settings.VOTABLE_NODE: worksheet.write_string(row,col, 'Votable') elif node.nodetype == settings.CLUSTER: worksheet.write_string(row,col, 'Cluster') col += 1 # fillcolor = models.CharField() if node.fillcolor: worksheet.write_string(row,col, '#' + node.fillcolor[:6]) col += 1 # clusterid = models.PositiveIntegerField() if node.clusterid: worksheet.write_number(row,col, node.clusterid, cell_integer_format) # 'Group' cell name used by openmappr.org col += 1 if node.clusterid: worksheet.write_number(row,col, node.clusterid, cell_integer_format) # 'Cluster' cell name used by Kumu.io col += 1 else: col += 2 # created_date = models.DateTimeField() if node.created_date: worksheet.write_string(row,col, str(node.created_date)) col += 1 # owner = models.ForeignKey(User) if node.owner: worksheet.write_number(row,col, node.owner_id, cell_integer_format) col += 1 # targetmapname = models.CharField() if node.targetmapname: worksheet.write_string(row,col, node.targetmapname) col += 1 # tunnelfarendmap = models.ForeignKey(Map_desc) if node.tunnelfarendmap: worksheet.write_number(row,col, node.tunnelfarendmap_id, cell_integer_format) col += 1 # status = models.CharField(max_length=1) if node.status: worksheet.write_string(row,col, node.status) col += 1 # xpos = models.FloatField() if node.xpos: worksheet.write_number(row,col, node.xpos, cell_float_format) col += 1 # ypos = models.FloatField() if node.ypos: worksheet.write_number(row,col, node.ypos, cell_float_format) col += 1 # size = models.CharField(max_length=13, default='2.5,2.5') if node.size: worksheet.write_string(row,col, str(node.size)) col += 1 # lwidth = models.FloatField() if node.lwidth: worksheet.write_number(row,col, node.lwidth, cell_float_format) col += 1 # shape = models.SmallIntegerField() if node.shape: worksheet.write_string(row,col,convertToTextShape(node.shape)) col += 1 # style = models.CharField() if node.style: worksheet.write_string(row,col, node.style) col += 1 # pencolor = models.CharField() if node.pencolor: worksheet.write_string(row,col, node.pencolor) col += 1 # fontcolor = models.CharField() if node.fontcolor: worksheet.write_string(row,col, node.fontcolor) col += 1 # numcomments = models.PositiveSmallIntegerField(default=0) if node.numcomments: worksheet.write_number(row,col, node.numcomments, cell_integer_format) col += 1 # url = models.URLField() if node.url: worksheet.write_string(row,col, node.url) col += 1 # description = models.CharField(max_length=settings.NODE_DESCRIPTION_LENGTH, blank=True, null=True, default=None) if node.description: worksheet.write_string(row,col, node.description) # prepare for next row row += 1 col = 0 # A second worksheet for edges worksheet = workbook.add_worksheet('links') columnHeaders = ( ['ID','From','To','Source','origin','Target','Label','Color','edgecost', 'penwidth'],) row = 0 col = 0 # Write column headers of edge worksheet for header in columnHeaders[0]: worksheet.write_string(row, col, header) col += 1 # Write edge data to edge worksheet row = 1 col = 0 edgeData = Edge.objects.filter(ofmap_id=mapId, status=settings.EDGE_ACTIVE).select_related('origin','target') for edge in edgeData: worksheet.write_number(row,col, edge.id, cell_integer_format) # ID col += 1 if edge.origin.label: worksheet.write_string(row,col, edge.origin.label) # From (used by Kumu.io) else: worksheet.write_string(row,col, ' ') # From (used by Kumu.io) col += 1 if edge.target.label: worksheet.write_string(row,col, edge.target.label) # To (used by Kumu.io) else: worksheet.write_string(row,col, ' ') # To (used by Kumu.io) col += 1 if edge.origin: worksheet.write_number(row,col, edge.origin_id) # Source (used by openmappr.org) col += 1 if edge.origin: worksheet.write_number(row,col, edge.origin_id) # origin (used by Ideatree) col += 1 if edge.target: worksheet.write_number(row,col, edge.target_id) # Target (used by openmappr.org) col += 1 if edge.label: worksheet.write_string(row,col, edge.label) # Label (of edge) col += 1 if edge.color: worksheet.write_string(row,col, '#' + edge.color[:6]) # Color col += 1 if edge.cost: worksheet.write_number(row,col, edge.cost, cell_integer_format) # Edge cost col += 1 if edge.penwidth: worksheet.write_number(row,col, edge.penwidth, cell_float_format) # Penwidth row += 1 col = 0 except: raise def _makeNodesFromImport(request, importedNodes, provisionalNodeIdList): try: # Convert values and insert needed attributes for nd in importedNodes: # Translation from Ideatree export FIXME: Ideatree uses 'fillcolor' on the server and 'color' on the client. if nd.get("fillcolor"): nd["color"] = nd["fillcolor"] if nd.get("nodetype"): if nd["nodetype"].lower() == "node": # FIXME: define as 'long type names' in settings, and use here and in worksheetToList() nd["nodetype"] = settings.ORIGINAL_NODE elif nd["nodetype"].lower() == "tunnel": nd["nodetype"] = settings.TUNNEL_NODE elif nd["nodetype"].lower() == "votable": nd["nodetype"] = settings.VOTABLE_NODE elif nd["nodetype"].lower() == "cluster": nd["nodetype"] = settings.CLUSTER else: # default if no item type supplied nd["nodetype"] = settings.ORIGINAL_NODE # FIXME standardize with client on using either xpos/ypos or pos, but not both! # FIXME: only here to try to coax client to placing nodes. If it doesn't work, remove from here. #if not 'pos' in nd and ('xpos' in nd and 'ypos' in nd): #nd["pos"]= str(nd["xpos"])+","+ str(nd["ypos"]) for attr in nd: # FIXME: reconsider whether this is necessary and effective if not re.match(settings.STRING_SAFETY_REGEX, attr): raise Exception("Illegal characters found. Only AlphaNumberic characters are allowed, for security purposes.") if not provisionalNodeIdList: raise Exception("_makeNodesFromImport: ran out of provisional node ids. Cannot continue, so aborting.") nd["nodeID"] = str(provisionalNodeIdList.pop()) # update the clusterid to an actual allocated provisional id corresponding to a cluster. if nd.get("clusterid"): nd["clusterid"] = [nodeDict["nodeID"] for nodeDict in importedNodes if nodeDict["id"]==nd["clusterid"]][0] nd["action"]="createNode" # A kludge to get by the action setting in issueJsExec() # FIXME: raise an exception here and see if it really fails except: raise def _makeEdgesFromImport(request, importedNodes, importedEdges, provisionalEdgeIdList): try: # node IDs from the imported file are no longer valid, since we've reassigned them to node provisionalIds, so we have to match up # the edges with the previous node ids, and then re-assign edge ids to their newly allocated provisionalIds. # NOTE: requires lower case 'id','origin','target' # FIXME: how much of this is just messy name conversion from Kumu or from inconsistent naming within IdeaTree? from numbers import Number for edg in importedEdges: origin=None # Security checks # FIXME: fold these into an excel format checker if edg.get("origin") and isinstance(edg["origin"], Number): # edgge endpoints are given as node ids edg["origin"] = int(edg["origin"]) if edg.get("target") and isinstance(edg["target"], Number): # edge endpoints are given as node ids edg["target"] = int(edg["target"]) # Now that we have an origin id - unless edges are specified by node label endpoints - we can update the origin with a provisional node id. # FIXME hard coded to depend on change message format foundOrigin=False foundTarget=False for importedNode in importedNodes: if importedNode.get("id") and edg.get("origin") and (importedNode["id"] == edg["origin"] or importedNode["label"] == edg["origin"]): # the node id in the import file, corresponding to the edge id in the import file edg["origin"] = importedNode["nodeID"] # the newly assigned node id in a previous step foundOrigin=True elif importedNode.get("id") and edg.get("target") and (importedNode["id"] == edg["target"] or importedNode["label"] == edg["target"]): # the node id in the import file, corresponding to the edge id in the import file edg["target"] = importedNode["nodeID"] # the newly assigned node id in a previous step foundTarget=True if foundOrigin==False: edg["origin"]=None msg = 'During Import: No origin found for edge id='+str(edg["id"]) + ' Looking for:' + str(edg["origin"]) print(msg, file=sys.stderr) raise Exception(msg) if foundTarget==False: edg["target"]=None msg = 'During Import: No target found for edge id='+str(edg["id"]) + ' Looking for:' + str(edg["target"]) print(msg, file=sys.stderr) raise Exception(msg) if not (foundOrigin and foundTarget): # print('Edge id='+str(edg["id"]) + ' deleted.', file=sys.stderr) importedEdges = [e for e in importedEdges if e["id"] != edg["id"]] else: # Take an allocated edge from the list of provisionals edg["edgeId"]= str(provisionalEdgeIdList.pop()) edg["action"]="createEdge" # An ugly kludge to get by the action setting in issueJsExec() except: raise # FIXME: defunct def _worksheetToElemCount(sheet, elemType): try: # First, convert into more readable dict form. elems = {} elemCount = 0 for rowx in range(sheet.nrows): if rowx > 0: # row zero is for column names, by convention elemCount += 1 return {elemType:elemCount} # FIXME: raise an exception here and see if it really fails except: raise TAG_RE = re.compile(r'<[^>]+>') def remove_tags(text): # FIXME: make this able to be called in sequence: replace().remove_tags() return TAG_RE.sub('', text) #def _csvToList(request, mapId, userId, csvfile, fieldList): # try: # # First, convert into more readable dict form. # elemList = [] # for row in csvfile: # elemDict = {} # if csvfile.line_num == 1: # colnames = row # else: # ndx = 0 # for value in row: # if isinstance(value, str): # value = value.replace(' ','') # Remove extraneous inserted spaces # # Remove tags (inserted by Kumu, if not others) and front/back whitespace, for security, and also to allow matching up of edge 'from/to' to node labels # value = remove_tags(value).strip() # elemDict.update({colnames[ndx]:value}) # ndx += 1 # if elemDict: # elemList.append(elemDict) # return elemList # # FIXME: raise an exception here and see if it really fails # except: # raise def _worksheetToList(request, mapId, userId, sheet, fieldList): try: # First, convert into dict form. itemList = [] masterClusterList = [] childClusterList = [] colnames = [] for c in sheet.iter_rows(min_row=1, max_row=1, values_only=True): col = c colnames = [name.lower() for name in col] for rowx in range(2,sheet.max_row + 1): rowDict = {} cellvals = [] for row in sheet.iter_rows(min_row=rowx, max_row=rowx, values_only=True): for cell in row: cellvals.append(cell) for ndx, val in enumerate(cellvals): if val or val==0: # discard empty fields, zero counts as not empty. if colnames[ndx] in fieldList: # discard fields we don't use in Ideatree # Clean up fields if isinstance(val, str): val = val.replace(' ','') # Remove extraneous inserted spaces # Remove tags (inserted by Kumu, if not others) and whitespace, for security, and also to allow matching up of edge 'origin/target' to node labels val = remove_tags(val).strip() val = myutils.truncateStr(val, settings.NODE_LABEL_LENGTH) # Additional security check if not re.match(settings.STRING_SAFETY_REGEX, val): raise Exception("Illegal characters found in imported node list. Only AlphaNumberic characters are allowed, for security purposes.") else: val = int(val) # Sometimes '1.0' is sent instead of '1', so convert. Which means floats are expected to be strings. rowDict.update({colnames[ndx]:val}) if rowDict: if rowDict.get("nodetype") and rowDict["nodetype"].lower()=='cluster' and not rowDict.get("clusterid"): masterClusterList.append(rowDict) elif rowDict.get("nodetype") and rowDict["nodetype"].lower()=='cluster' and rowDict.get("clusterid"): childClusterList.append(rowDict) else: itemList.append(rowDict) # NOTE: clusters must go first so client has something to insert nodes into if the node asks for it. # Also, clusters must be in order of nesting starting with the root. if masterClusterList and childClusterList: masterClusterListIds = [item["id"] for item in masterClusterList] # Extract parent cluster parameter from each child cluster childClusterParentIds = [item["clusterid"] for item in childClusterList] findClusterParent(masterClusterListIds, childClusterParentIds, childClusterList, masterClusterList, 0) return masterClusterList + itemList # FIXME: raise an exception here and see if it really fails except: raise def findClusterParent(masterClusterListIds, childClusterParentIds, childClusterList, masterClusterList, ndx): # Arrange clusters in the proper order in a master list by moving child clusters to the master list after the parents. try: if ndx < len(childClusterParentIds): # there are more to process id = childClusterParentIds[ndx] if id in masterClusterListIds: # the parent cluster exists, so we can add the child. currentChild = [clust for clust in childClusterList if clust["clusterid"]==id][0] masterClusterList.append(currentChild) #childClusterParentIds.remove(id) # prevent it being found next go-around if ndx < len(childClusterParentIds): # there are more to process ndx += 1 findClusterParent(masterClusterListIds, childClusterParentIds, childClusterList, masterClusterList, ndx) #else: # final one #id = childClusterParentIds[0] #currentChild = [clust for clust in childClusterList if clust["clusterid"]==id][0] #masterClusterList.append(currentChild) #childClusterParentIds.remove(id) return except: raise def _edgesToList(request, mapId, userId, sheet, fieldList): try: # First, convert into more readable dict form. elemList = [] firstDataRow = 1 for rowx in range(sheet.nrows): #print("processing row:" + str(sheet.row_values(rowx))) elemDict = {} hasColnames = [col for col in sheet.row_values(0) if isinstance(col, str)] if not hasColnames: hasColnames = ['origin', 'target'] firstDataRow = 0 if rowx >= firstDataRow: # get row values cellvals = sheet.row_values(rowx) oneEdgeDict = dict(zip(fieldnames, row)) for ndx, cv in enumerate(cellvals): if cv and colnames[ndx] in fieldList: value = cv if isinstance(value, str): value = value.replace(' ','') # Remove extraneous inserted spaces # Remove tags (inserted by Kumu, if not others) and front/back whitespace, for security, and also to allow matching up of edge 'from/to' to node labels value = remove_tags(value).strip() elemDict.update({colnames[ndx]:value}) if elemDict: elemList.append(elemDict) return elemList # FIXME: raise an exception here and see if it really fails except: raise @login_required() @user_passes_test(not_guest_user) def getFileUploadForm(request): try: format = request.POST.get("format") if format: format=format.lower() form = FileUploadForm(request.POST) if not form.is_valid(): raise Exception("Invalid FileUploadForm.") context = {'form':form, 'format':format, 'needsProvisionals':1, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request, 'fileuploadform.html', context ) except Exception as err: return HttpResponse(str(err), status=406) @login_required() @user_passes_test(not_guest_user) def uploadFile(request): try: mapId = currently_opened_map(request) if request.method == 'POST': form = FileUploadForm(data=request.POST, files=request.FILES) # FIXME: if the file name is changed but the same file chooser is up, an exception results which doesn't get shown in a popup. if not form.is_valid(): raise Warning(form.errors) # FIXME: format this better else: format = request.POST.get("format") if format: format=format.lower() filename, file_extension = os.path.splitext(request.FILES['file_source'].name) if file_extension[1:].lower() != format: raise Warning("The file extension doesn't match the type you selected from the Import menu.
Please select a file with '"+format+"' extension.") if format not in settings.VALID_IMPORT_FORMATS: return HttpResponse("Not a valid import file format.", status=406) importDir = make_absoluteImportDir(mapId) # FIXME: put both graphviz suffixes in one constant if format in [settings.GRAPHVIZ_GV.lower(), settings.GRAPHVIZ_DOT.lower()]: format = settings.GRAPHVIZ_DOT.lower() # FIXME: "/uploaded" is not DRY # FIXME: use os.join: here and elsewhere similar writeTo = importDir + "/uploaded." + format with open(writeTo, 'wb+') as destination: for chunk in request.FILES['file_source'].chunks(): destination.write(chunk) return getImport(request) # NOTE: chains to get count of needed provisional elements, see "STEP 1" except FileNotFoundError as err: return JsonResponse({'message':"FileNotFoundError:" + str(err)}, status=406) except Warning as err: return JsonResponse({'message':str(err)}, status=406) except Exception as err: return JsonResponse({'error':str(err)}, status=406) def makeCommands(request,mapId,userId,nodeList,edgeList,provisionalNodeIds,provisionalEdgeIds): try: _makeNodesFromImport(request,nodeList,provisionalNodeIds) for node in nodeList: try: if 'tunnelfarendmap' in node: print("TUNNEL FAR END" + str(node["tunnelfarendmap"])) targetMap = node["tunnelfarendmap"] thisId = node.get("id") have_map_access(request, targetMap) except PermissionDenied: # purge dangling tunnels to maps the user doesn't have access to. nodeList = [d for d in nodeList if d.get('id') != thisId] continue # rename 'importedEdges' # FIXME: this uses the mechanism that context menus use, while elsewhere we send a change message transaction. Do it one way! _makeEdgesFromImport(request,nodeList,edgeList,provisionalEdgeIds) for node in nodeList: node.pop('id') # defined by the spreadsheet, but just confuses things once purging of dangling tunnels is done. for edge in edgeList: edge.pop('id') # defined by the spreadsheet, but just confuses things once purging of dangling edges is done. # Now we don't need the node id fields anymore to match up edges up with provisional endpoint node IDs, so delete them. cmnds = issueJsExec(nodeList, forAppend=False) edgeCmnds = issueJsExec(edgeList, forAppend=True) for edge in edgeCmnds: cmnds["execute"].append(edge) return(cmnds) except: raise def countSpreadsheetRows(sheet): try: rows = sheet.iter_rows(values_only=True) neededItems = 0 for row in rows: if row[0] != None: neededItems += 1 neededItems -= 1 # Not including the header row return(neededItems) except: raise def makeNumericEndpoints(nodeList, edgeList): for edge in edgeList: for node in nodeList: if edge["origin"]==node["label"]: edge["origin"]=node["id"] elif edge["target"]==node["label"]: edge["target"]=node["id"] return edgeList def makeTransactionWithData(mapId,data): try: dataStr = makeTransactionHeaderJSON(mapId) dataStr = myutils.rStrChop(dataStr, '[') dataStr += json.dumps(data) dataStr += "}" return(dataStr) except: raise def writeFileToBePulled(mapId, outfilePath, data): try: dataStr = makeTransactionWithData(mapId,data) chngMsgs_fp = open(outfilePath,"wt", encoding="UTF-8") # FIXME: enforce semaphore chngMsgs_fp.write(dataStr) chngMsgs_fp.close() except: raise @login_required() @user_passes_test(not_guest_user) def getImport(request): try: userId = int(request.session["_auth_user_id"]) mapId = currently_opened_map(request) if request.method == "POST" and request.POST.get("format"): format = request.POST.get("format") if format: format=format.lower() importDir = make_absoluteImportDir(mapId) # FIXME: use os.path.join: if format in [settings.GRAPHVIZ_GV.lower(), settings.GRAPHVIZ_DOT.lower()]: format = settings.GRAPHVIZ_DOT.lower() importedFile = importDir + "/uploaded." + format # STEP 1 # request for this is built into fileuploadform.html if request.POST.get("countNeededProvisionalElements"): if format in [settings.GRAPHVIZ_GV.lower(), settings.GRAPHVIZ_DOT.lower()]: layoutResultFile = makeLayoutFile(request, haveGraphvizSrc=True) result = parseLayout(request, layoutResultFile, countOnly=True) result["format"]=format return HttpResponse(json.dumps(result), status=200) elif format == settings.EXCEL_FILE_SUFFIX.lower(): from openpyxl import load_workbook workbook = load_workbook(filename=importedFile) neededNodes = countSpreadsheetRows(workbook.worksheets[0]) # By convention, nodes are on sheet 0 neededEdges = countSpreadsheetRows(workbook.worksheets[1]) # By convention, edges are on sheet 0 # FIXME use consistent naming throughout: 'provisionalNodes' return HttpResponse(json.dumps({'nodes':neededNodes, 'edges':neededEdges, 'format':format})) else: raise Exception("Unknown format in getImport step 1.") # STEP 2, second call from client. if request.POST.get("provisionalElements"): provElems = json.loads(request.POST.get("provisionalElements")) provNodeIds = provElems["nodeIds"] provEdgeIds = provElems["edgeIds"] if format in [settings.GRAPHVIZ_GV.lower(), settings.GRAPHVIZ_DOT.lower()]: # FIXME: possible optimization: propagate a transactionId between step1 and step2 of this def, and if they match, reuse the # layout file generated in step1. A timestamp is not the best way, but here's a way to generate one: https://forum.graphviz.org/t/how-to-timestamp-dot-fdp-neato-twopi-circo-v-output/654 layoutResultFile = makeLayoutFile(request, haveGraphvizSrc=True) result = parseLayout(request, layoutResultFile, countOnly=False, provisionalNodeIds=provNodeIds, provisionalEdgeIds=provEdgeIds, makeItems=True) importPath = make_absoluteImportDir(mapId) + '/changeMessages.json' # FIXME use os.path.join writeFileToBePulled(mapId, importPath, result) return HttpResponse("OK", status=200) # FIXME: move out into its own def or module. # FIXME: can this return change messages instead of js commands? Then all imports work the same. # FIXME: further, if context menus can return change messages, a huge simplification # FIXME: test user error message when an EXCEL file that's not a map export is imported. if format == settings.EXCEL_FILE_SUFFIX.lower(): from openpyxl import load_workbook workbook = load_workbook(filename = importedFile, data_only=True) sheet = workbook.worksheets[0] # By convention, nodes are on sheet 0 # Get the Node field names we use in Ideatree fieldList = [f.name.lower() for f in Node._meta.get_fields()] # FIXME URGENT: does this include 'id', which we don't want? fieldList.append("type") # we'll translate this into our own type name, like 'nodetype' fieldList.append("color") # we'll translate this into our own type name, like 'fillcolor' fieldList.append("id") # we'll translate this into our own type name fieldList.append("name") # we'll translate this into our own type name # rename 'importedNodes' nodeList = _worksheetToList(request, mapId, userId, sheet, fieldList) # FIXME: this uses the simpleajax/executeServerCommands mechanism that context menus use, # while in initiatePull (layout and prettify) we send a change message transaction. Do it one way! # Now edges #fieldList = [f.name.lower() for f in Edge._meta.get_fields()] # FIXME better than the below. fieldList = ['id','label','color', 'edgecost','penwidth', 'origin','target'] # must all be lower case sheet = workbook.worksheets[1] # By convention, edges are on sheet 1 edgeList = _worksheetToList(request, mapId, userId, sheet, fieldList) # Convert to a command the js client will use to a) generate a request, b) enable undo, and c) send back to server for db saving. # FIXME: does the client really need a json of a list? #return HttpResponse(json.dumps([makeCommands(request,mapId,userId,nodeList,edgeList,provNodeIds,provEdgeIds)])) edgeList = makeNumericEndpoints(nodeList, edgeList) layoutResultFile = makeLayoutFile(request, haveGraphvizSrc=False, nodeListOfDicts=nodeList, edgeListOfDicts=edgeList) result = parseLayout(request, layoutResultFile, countOnly=False, provisionalNodeIds=provNodeIds, provisionalEdgeIds=provEdgeIds, makeItems=True) importPath = make_absoluteImportDir(mapId) + '/changeMessages.json' # FIXME use os.path.join writeFileToBePulled(mapId, importPath, result) return HttpResponse("OK", status=200) # JS initiatePull reads the data from a server file. Can it be streamed instead? msg = {'error':"Error: This file format is not supported."} return HttpResponse(json.dumps(msg), status=200) # FIXME: what are the possible csv module exceptions? # FIXME: test all warnings and exceptions according to this model. except Warning as err: return JsonResponse({'message':str(err)}, status=406) except Exception as err: return JsonResponse({'error':str(err)}, status=406) @login_required() @user_passes_test(not_guest_user) def publish(request, mapId): try: graphtitle = str(Map_desc.objects.get(pk=mapId)) publishFile = make_publishPath(mapId, settings.PUBLISH_FILE_SUFFIX, absolute=False) absPublishFile = make_publishPath(mapId, settings.PUBLISH_FILE_SUFFIX, absolute=True) if os.path.exists(absPublishFile): return render(request, "publish.html", context={'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED,'publishResultFile':publishFile, 'graphtitle':graphtitle}) else: return render(request, "publishEmpty.html", context={'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'graphtitle':graphtitle}) except Exception as err: return HttpResponse(str(err), status=406) @login_required() @user_passes_test(not_guest_user) def unpublish(request): try: mapId = currently_opened_map(request) graphtitle = str(Map_desc.objects.get(pk=mapId)) publishFile = make_publishPath(mapId, settings.PUBLISH_FILE_SUFFIX, absolute=True) context={'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED, 'graphtitle':graphtitle } if os.path.exists(publishFile): try: os.remove(publishFile) except OSError: raise else: return render(request, "publishFileNotFound.html", context) return render(request, "unPublishSuccess.html", context) except OSError as err: return HttpResponse(str(err), status=406) except Exception as err: return HttpResponse(str(err), status=406) @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() @user_passes_test(not_guest_user) def export(request): try: if request.POST.get("step1"): return render(request,'ideatree/publishAreYouSure.html', context={'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED }) userId = int(request.session["_auth_user_id"]) exportFormat = request.POST.get("exportFormat") mapId = currently_opened_map(request) if exportFormat in [settings.GRAPHVIZ_GV.lower(), settings.GRAPHVIZ_DOT.lower()]: checkPermissions(request, userId,"exportDot", mapId) # FIXME: DRY: dotOutputFile = get_absoluteExportPath(mapId, exportFormat) orientation = request.POST['orientation'] if request.POST.get('orientation') else lookup_map_info(mapId,"orientation") layoutengine = request.POST['layoutengine'] if request.POST.get('layoutengine') else lookup_map_info(mapId,"layoutengine") Graph2dotFile(mapId, orientation, layoutengine, dotOutputFile, None, None, exportFormat, exporting=True) elif exportFormat == settings.PUBLISH_FILE_SUFFIX.lower(): checkPermissions(request, userId,"publish", mapId) dotInputFile = make_absoluteOutfilePath(mapId,settings.GRAPHVIZ_GV) orientation = request.POST['orientation'] if request.POST.get('orientation') else lookup_map_info(mapId,"orientation") layoutengine = request.POST['layoutengine'] if request.POST.get('layoutengine') else lookup_map_info(mapId,"layoutengine") Graph2dotFile(mapId, orientation, layoutengine, dotInputFile, None, None, exportFormat, exporting=True) # The current ps2pdf program can't handle utf-8, so we have to convert the data.. # NOTE: shell=True only usable securely because no user-input is part of the call. # FIXME: use run() or system() instead? # NOTE: These shell commands run as wsgiuser even when Apache is the webserver. # for PDF only: out = check_output(['iconv -c -f UTF8 -t LATIN1//TRANSLIT -o ' + dotInputFile + ' < ' + dotInputFile], shell=True) # Make the filename publishResultFile = make_publishPath(mapId, settings.PUBLISH_FILE_SUFFIX) graphvizLayout(mapId, dotInputFile, publishResultFile, layoutengine) # The 'linkToPublished' context is what will be clicked to route back through urls.py, to call def publish(), to finally render the pdf. return render(request,'ideatree/publishLink.html', context={"linkToPublished":os.path.join("publish",str(mapId))+" ", 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} ) elif exportFormat == settings.EXCEL_FILE_SUFFIX.lower(): checkPermissions(request, userId,"exportEXCEL", mapId) exportResultFile = get_absoluteExportPath(mapId,settings.EXCEL_FILE_SUFFIX) import xlsxwriter with xlsxwriter.Workbook(exportResultFile) as workbook: makeExcelFile(request, mapId, workbook) else: raise Exception("Unsupported or missing format.") context = {'action':'download/', "format":exportFormat} return render(request, "ideatree/download.html", context, content_type="text/html") # This just to get filepath into the request. # FIXME: what are the possible xlsxWriter exceptions? except CalledProcessError as err: myutils.writeErr(err) raise Exception(err) except OSError as err: myutils.writeErr(err) raise Exception(err) except Warning as err: return JsonResponse({'message':str(err)}, status=406) except Exception as err: myutils.writeErr(err) return JsonResponse({'sysErr':str(err)}, status=406) class Graph2dotFile(): __graphSpec = "" # NOTE: TEMPINVISFLAG was used to create temporary id for the bogus node inserted in every cluster to enable graphviz to plot edges that connect to a cluster. # However, it's a node that the server makes up without reference to either the db or imported source, thus there's no corresponding provisional # object on the client. Also, it complicates parseLayout(). Even without it, the edge to a cluster may not have the right path, but moving # the cluster will recalculate that immediately. Maybe there's a way to automate moving it one pixel after doing an import or layout. __TEMPINVISFLAG = settings.CLUSTER_INVISIBLE_NODE_FOR_EDGE_PLACEMENT __clusterList = [] __hidden_nodes = [] __branchHeads = [] __totalNodes = 0 # FIXME: how to make this private inside of Nodes2dot? Class within class? def __init__(self, mapId, orientation, layoutengine, dotFile, nodeListOfDicts=None, edgeListOfDicts=None, exportFormat=None, exporting=False): try: # FIXME: why does 'self.__layoutengine' require 'self._Graph2dotFile__layoutengine' in pdb, yet works in the code. Or does it? self.__mapId = mapId self.outfile = dotFile self.nodeListOfDicts = nodeListOfDicts self.edgeListOfDicts = edgeListOfDicts self.exporting = exporting # FIXME: serves any purpose that you can't get by checking exportFormat? self.layoutengine = layoutengine self.exportFormat = exportFormat self.graphDefaults4Layout(orientation) self.nodes2dot() # IMPORTANT: nodes must be processed before edges in order to get a list of clusters and hidden nodes. self.edges2dot() self.__graphSpec += "}\n"; # close out the graph # *********** and write the output file ************************* # FIXME: write test with international characters dot_fp = open(self.outfile,"wt", encoding="UTF-8") dot_fp.write(self.__graphSpec) dot_fp.close() except FileNotFoundError as err: raise Exception("FileNotFoundError:" + str(msg)) except: raise def graphDefaults4Layout(self, orientation): try: # FIXME: Does the client make use of node fontsize? If so, it should be sent by the client, stored per map, and sent as a default in makeGraphDefaults() fontsize = settings.NODEFONTSIZE defaultEdgePenwidth = settings.DEFAULTEDGE_PENWIDTH defaultEdgePencolor = settings.DEFAULTEDGE_PENCOLOR # NOTE: ps2pdf doesn't understand DejaVuSans, while prior to that, graphviz understands how to convert NimubusSans to DejaVuSans.. fontp = "NimbusSans-Bold" if (self.exporting and self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()) else "DejaVuSans.ttf" # NOTE: though map loader sends default node width,height to the client, here we rely on the database values for individual nodes, not on defaults. # FIXME: does the client really need them as default values? # NOTE: splines and sep arguments slow down layout # NOTE: don't use ordering="out". will cause dot to abort for some graphs! aspectratio = "ratio=\"compress\"" if (self.exporting and self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()) else " " # NOTE: Unfortunately, graphviz will put graph labels on clusters, too, so they can't be used for the graph. heightInInches = 6.2 # at 72 dpi #self.heightInInches = 4.6 # at 96 dpi # limit size in inches (pdf has a limit of 199,199) # See https://stackoverflow.com/questions/14784405/how-to-set-the-output-size-in-graphviz-for-the-dot-format size = " size=\"13.3,"+str(heightInInches)+"\" " if (self.exporting and self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()) else " " # NOTE: This is a crucial setting and will make a huge difference in the layouts. fixedsize = " fixedsize=\"false\"" if (self.exporting and self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()) else " fixedsize=\"true\"" # Make source file header and defaults for nodes and edges. # NOTE: bottom-to-top orientation flips everything, so if you want label at the top, you have to say you want it at the bottom: dpi = str(settings.DOTS_PER_INCH) splines = " splines=\"line\" " if self.layoutengine == settings.CIRCULAR_LAYOUT_ENGINE else " splines=\"spline\" " # FIXME: compare run time of voronoi vs. scale for overlap overlap = " overlap=\"voronoi\" " if self.layoutengine in [settings.CIRCULAR_LAYOUT_ENGINE] else " overlap=\"prism\" " if self.layoutengine == settings.RADIAL_LAYOUT_ENGINE: overlap = " overlap=\"scale\" " ranksep = " ranksep=\"4.0\" " if self.layoutengine == settings.RADIAL_LAYOUT_ENGINE else " ranksep=\"1.0\" " inputscale = " inputscale=\"" + str(settings.DOTS_PER_INCH) + "\" " sep=" sep=\"+16.0\" " # In points? Space between nodes when overlap removed. Not for dot. edgesep = " esep=\"+12.0\" " # Should be less than sep. Not for dot. In points? nodesep=" nodesep=\"0.50\" " # In inches. Space between nodes in the same rank. nodemargin = " margin=\"0.05\" " # in inches edgeweight = " weight=\"300\" " # Only for purposes of making len be honored by graphviz. Separate mechanism does weights (called 'cost') for shortest path analysis. edgelen = " len=\"0.5\" " # In inches pin = " pin=\"true\" " # Pin is set knowing that only fdp and neato will honor it. mindist = " mindist=\"2.0\" " # circo only. In inches? docs don't say. graphmargin = " " # in inches graphpad = " pad=\"0.5\" " # in inches (fdp and neato only) # FIXME: graphviz native output requires a fontsize almost four times larger to equal the font size displayed by the client. fontsize = " fontsize =\"42.0\"" if (self.exporting and self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()) else " fontsize=\""+ settings.NODEFONTSIZE + "\" " edgefontsize = " fontsize =\"18.0\"" if (self.exporting and self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()) else " fontsize=\""+ settings.EDGEFONTSIZE + "\" " self.__graphSpec += "digraph IdeaTree {\ngraph [ notranslate=\"true\" " + mindist + graphmargin + graphpad + size + inputscale + " compound=\"true\" concentrate=\"false\" " + nodesep + sep + edgesep + ranksep + " maxiter=\"500\" " + overlap + aspectratio + " truecolor=\"false\" dpi=\""+dpi+"\" " + splines +" rankdir=\"" + orientation + "\"]\n" self.__graphSpec += "node [" + pin + nodemargin + fixedsize + fontsize + " fontname=\"" + fontp + "\"]\n" # FIXME: graphviz arrowsize is larger than client renders it. arrowheadsize = " arrowsize=\"2.0\" " if (not self.exporting and not self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()) else " arrowsize=\"3.0\" " self.__graphSpec += "edge [" + edgeweight + arrowheadsize + edgelen + " arrowhead=\"normal\" " + edgefontsize + " labeldistance=\"1.5\" tailclip=\"true\" headclip=\"true\" color=\"#" + defaultEdgePencolor +"\" penwidth=\"" + defaultEdgePenwidth +"\" fontname=\""+ fontp + "\"]\n" return(True) except: raise def nodes2dot(self): def dot4Anode(node): try: label = node.label if hasattr(node,'label') else None if self.exporting and node.label==settings.EMPTY_NODE_LABEL_PLACEHOLDER: label = " " shape = " shape=\"" + convertToTextShape(node.shape) + "\"" if hasattr(node,'shape') else "" style = " style=\"" + node.style +"\"" if hasattr(node,'style') else "" # FIXME: client sets its own fontcolor for Shortest Path endpoint nodes. Good to check for readability of font color, which I think it does, # but it shouldn't decide colors, only alter them, so that server db is canonical. fontcolor = " fontcolor=\"#" + node.fontcolor + "\"" if hasattr(node,'fontcolor') else "" #NOTE: if not set, graph defaults apply fillcolor = " fillcolor=\"#" + node.fillcolor + "\"" if hasattr(node,'fillcolor') else "" pencolor = " pencolor=\"#" + node.pencolor + "\"" if hasattr(node,'pencolor') else "" target = "_blank" if hasattr(node,'nodetype') and (node.nodetype != None) else "_self" # FIXME URGENT: tunnelfarendmap_id is not set when a tunnel target is created, but works anyway. #url = "?mid=" + str(node.tunnelfarendmap_id) if(node.nodetype == settings.TUNNEL_NODE) else node.url # FIXME: where is mid= used? url = node.url if hasattr(node,'url') and node.url != None and hasattr(node,'nodetype') and (node.nodetype != settings.TUNNEL_NODE) else None href = " URL=\"" + url + "\" target=\"" + target + "\"" if url else "" if hasattr(node,'xpos') and hasattr(node,'ypos'): # pos relates to screen values meaningful to the js client, not to export formats. #if (self.exporting and self.exportFormat==settings.PUBLISH_FILE_SUFFIX.lower()): #totalYHeightPts = self.heightInInches * settings.DOTS_PER_INCH * 0.75 #ypos = totalYHeightPts - node.ypos #else: #ypos = node.ypos pos = ' pos="' + str(node.xpos) + ',' + str(node.ypos) + '"' else: pos = " " widthAndHeight = "" if hasattr(node,'size'): size=node.size.split(',') widthAndHeight = " width=\""+ size[0].strip() + "\" height=\""+size[1].strip() + "\" " # This is actually minimum width/height if fixedsize=true if hasattr(node,'id'): self.__graphSpec += "\"" + str(node.id) +"\" [ ID=\""+ str(node.id) + "\"" + pos + widthAndHeight + shape + style + fontcolor + fillcolor + pencolor + " label=\""+ str(label) + "\"" + href + "]\n" return(True) except: raise # FIXME: not to be confused with makeLabel() def makeLabelTag(label): try: if label: label = ' label="'+ label + '"' else: label = "" return label except: raise def getClusterContents(clustId,label): # get nodes (and other clusters, which are a type of node) contained in this cluster try: self.__clusterList.append(clustId) # FIXME: only return the values needed nodes = Node.objects.filter(ofmap=self.__mapId, clusterid=clustId).exclude(status=settings.NODE_DELETED).exclude(status=settings.PROVISIONAL).order_by('created_date') for node in nodes: self.__totalNodes += 1 if (node.nodetype != settings.CLUSTER): dot4Anode(node) else: # nested cluster label = makeLabelTag(label) self.__graphSpec += "subgraph cluster" + str(node.id) + " {\n style=\"filled,rounded\" fillcolor=\"#" + node.fillcolor + "\" lwidth=\"" + settings.CLUSTER_LABEL_WIDTH_INCHES + "\" fontsize=\"" + settings.CLUSTER_LABEL_FONTSIZE + "\" ID=\"cluster"+ str(node.id) + "\"" + label + "\n" # FIXME In testing, edges to clusters seemed to work without this invisible node. Maybe not needed anymore? # FIXME fdp layout, used for pdf export, will draw edge to this invisible node rather than to the cluster border. #self.__graphSpec += "\"" + self.__TEMPINVISFLAG + str(node.id) + "\" [\n fixedsize=\"true\" ID=\""+ str(node.id) + "\" height=\"0\" width=\"0\" shape=\"point\"]\n" getClusterContents(node.id,node.label) #recursive self.__graphSpec += "}\n" # close out the cluster return(True) except: raise # ------------------------ MAIN of nodes2dot ---------------------------------------- try: # FIXME: only return the values needed # First, get top level clusters # TODO: here's where Excel import clusters will be detected, similar to below, "nodes not in any cluster". #if not self.nodeListOfDicts: clusters = Node.objects.filter(ofmap=self.__mapId, nodetype=settings.CLUSTER, clusterid=0).exclude(status=settings.NODE_DELETED).exclude(status=settings.PROVISIONAL).order_by('created_date') for clust in clusters: if(clust.hiddenbranch): self.__hidden_nodes.append(clust.id) label = makeLabelTag(clust.label) # No style given, since graphviz doesn't support the numeric styles we use: size=clust.size.split(',') widthAndHeight = " width=\""+ size[0].strip() + "\" height=\""+size[1].strip() + "\" " # This is actually minimum width/height if fixedsize=true self.__graphSpec += "subgraph cluster" + str(clust.id) + " {\n " + widthAndHeight + " style=\"filled,rounded\" fillcolor=\"#" + clust.fillcolor + "\" lwidth=\"" + settings.CLUSTER_LABEL_WIDTH_INCHES + "\" fontsize=\"" + settings.CLUSTER_LABEL_FONTSIZE + "\" ID=\"" + str(clust.id) + "\"" + label + "\n" # Insert an invisible node in every cluster so cluster-to-cluster edges have an endpoint. # FIXME: Later, when the client can render them, also include lhead and ltail to generate the right edge curves) # FIXME fdp layout, used for pdf export, will draw edge to this invisible node rather than to the cluster border. #self.__graphSpec += "\"" + self.__TEMPINVISFLAG + str(clust.id) + "\" [\n fixedsize=\"false\" ID=\"" + str(clust.id) + "\" height=\"0\" width=\"0\" shape=\"point\"]\n" getClusterContents(clust.id,clust.label) self.__graphSpec += "}\n" # close out the cluster # Now nodes not in any cluster if not self.nodeListOfDicts: nodes = Node.objects.filter(ofmap=self.__mapId, clusterid=0).exclude(status=settings.NODE_DELETED).exclude(nodetype=settings.PROVISIONAL).exclude(nodetype=settings.CLUSTER).order_by('-clusterid', 'created_date') else: nodes = self.nodeListOfDicts for node in nodes: if self.nodeListOfDicts: # FIXME If the model filter used above returned a values list this wouldn't be necessary, and subscripts could be used instead of dot notation. node = myutils.Struct(**node) if hasattr(node,'hiddenbranch') and node.hiddenbranch==True: self.__hidden_nodes.append(node.id) self.__totalNodes += 1 dot4Anode(node) if not self.__totalNodes: raise Warning("You can create a topic by dragging one of the icons at the upper right to this area.") return(True) except: raise nodes2dot.alters_data = True # FIXME URGENT why both this and edgeit()? edgit is for analyses and for map loads using changemessages, edges2dot for prettify and exports using graphviz dot. Combine. def edges2dot(self, engine=settings.DEFAULT_LAYOUT_ENGINE): try: if not self.edgeListOfDicts: edges = Edge.objects.filter(ofmap=self.__mapId, status=settings.EDGE_ACTIVE).values('id','origin','target','label','color','cost') else: edges = self.edgeListOfDicts for edge in edges: origin = edge['origin'] target = edge['target'] # FIXME: Why this line about radial layout?: cost = " cost=\"" + str(edge['cost']) + "\" " if (("cost" in edge) and self.layoutengine != settings.RADIAL_LAYOUT_ENGINE) else "" label = costToLabel(edge['cost'], edge['label']) if (("label" in edge) and self.exporting) else "" label = " label=\"" + label + "\" " color = " color=\"#" + edge['color'] + "\" " if ("color" in edge) else "" edgeHeadClipping = "" edgeTailClipping = "" if (target in self.__clusterList): # edge is to a cluster, not to a regular node. edgeHeadClipping = " lhead=\"" + str(target) +"\" " edgeHeadClipping = " lhead=\"cluster" + str(target) +"\" " target = str(target) #target = self.__TEMPINVISFLAG + str(target) if (origin in self.__clusterList): # edge is to a cluster, not to a regular node. edgeTailClipping = " ltail=\"" + str(origin) +"\" " origin = str(origin) #origin = self.__TEMPINVISFLAG + str(origin) # FIXME: __TEMPINVISFLAGxxxx nodes don't exist in the database, but they take their id from their parent cluster, so write a test for that. constraint="" style=" style=\"solid\" " if not self.exporting: collapsing = False # FIXME: not used if self.__hidden_nodes: if self.__branchHeads and (target in self.__branchHeads) and (origin not in self.__hidden_nodes): collapsing = True elif (target in self.__hidden_nodes) or (origin in self.__hidden_nodes): collapsing = True style=" style=\"invis\" " # FIXME: build this string from an object for better readability self.__graphSpec += "\"" + str(origin) + "\" -> \"" + str(target) + "\" [ origin=\"" + str(origin) + "\" target=\"" + str(target) + "\"" + edgeHeadClipping + " " + edgeTailClipping + " ID=\"" + str(edge['id']) + "\"" + constraint + " " + cost + " " + style + " " + color + " " + label + "]\n" return(True) except: raise edges2dot.alters_data = True def costToLabel(cost,label): if cost and int(cost) > 0: return "w"+str(cost) else: return "" return label def _duplicatesInList(seq): # From https://stackoverflow.com/questions/9835762/how-do-i-find-the-duplicates-in-a-list-and-create-another-list-with-them try: seen = set() seen_add = seen.add #to save lookup # adds all elements it doesn't know yet to seen and all other to seen_twice seen_twice = set( x for x in seq if x in seen or seen_add(x) ) return list(seen_twice) except: raise def _adjacentNodes(G): try: return ([n for n, nbrsdict in G.adjacency() for nbr, keydict in nbrsdict.items()]) except: raise @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() # See https://stackoverflow.com/questions/37353199/graphviz-vs-pygraphviz def analyze(request): try: import networkx as nx __mapId = currently_opened_map(request) outdir = make_absoluteOutfileDir(__mapId) name = str(__mapId) bigG = nx.MultiDiGraph() # FIXME: is not excluding clusters, maybe that's ok nodes = makenodes(request,__mapId, asNodeList=True, withLabel=False, excludeClusters=True) bigG.add_nodes_from([n.id for n in nodes]) del nodes positiveEdgeCostsOnly = True if (request.POST.get("algorithm")=="shortest_path" and request.POST.get("costed")=="costed") else False edges = edgeit(__mapId, endpointsAndEdgeCostsOnly=True, positiveEdgeCostsOnly=positiveEdgeCostsOnly) if request.POST.get("costed")=="costed": bigG.add_weighted_edges_from(edges) else: bigG.add_edges_from(edges) del edges if request.POST.get("algorithm")=="shortest_path": endpoints = MapPathEndpoints.objects.get(ofmap_id=__mapId) if not endpoints.pathBeginNode: raise Warning("Please right-click in the "+settings.WHAT_A_GRAPH_IS_CALLED +" and create a "+ settings.PATH_ORIGIN_LABEL +",
then connect it to one "+settings.WHAT_A_NODE_IS_CALLED + ".") elif not endpoints.pathEndNode: raise Warning("Please right-click in the "+settings.WHAT_A_GRAPH_IS_CALLED +" and create a "+ settings.PATH_END_LABEL +",
then connect it to one "+settings.WHAT_A_NODE_IS_CALLED + ".") #shortpath = nx.dijkstra_path(bigG, endpoints.pathBeginNode.id, endpoints.pathEndNode.id, "cost") shortpath = nx.dijkstra_path(bigG, endpoints.pathBeginNode.id, endpoints.pathEndNode.id) del bigG # From https://groups.google.com/forum/#!topic/networkx-discuss/87uC9F0ug8Y , post by Dan Schult. If too slow, consider pre-processing by converting to # DiGraph node by node along a path, choosing lighter weight edges when there are Multis. shortPathEdges = [z for z in zip(shortpath[1:],shortpath[:-1])] shortPathEdges = [Edge.objects.filter(target=s[0], origin=s[1], status=settings.EDGE_ACTIVE)[0] for s in shortPathEdges] allEdges = Edge.objects.filter(status=settings.EDGE_ACTIVE, ofmap_id=__mapId) canonicalTime= str(myutils.millisNow()) transID = str(random.randrange(1000000)) # FIXME: why not append the closing '}]' and convert to json here for easier manipulation? dataStr = makeTransactionHeaderJSON(__mapId, transID, canonicalTime) dataStr = myutils.rStrChop(dataStr, '[') # FIXME: correcting a design flaw? changesList = [] for edge in allEdges: # automatically reset all edge colors to the default. changesList.append(makeAlterEdgeChangeMessage(edge.id, "color", settings.DEFAULTEDGE_PENCOLOR)) for edge in shortPathEdges: # FIXME Can this be done with the **attr parameter to add_weighted_edges_from(), above? changesList.append(makeAlterEdgeChangeMessage(edge.id, "color", settings.EDGE_ANALYSIS_COLOR)) dataStr += json.dumps(changesList) dataStr += "}" return HttpResponse(dataStr, status=200, content_type="application/json", charset="UTF-8") raise Exception("No algorithm given.") except MapPathEndpoints.DoesNotExist: err = "Please right-click in the " + settings.WHAT_A_GRAPH_IS_CALLED + " and create a "+ settings.PATH_ORIGIN_LABEL +" and "+ settings.PATH_END_LABEL +",
then connect each to one "+settings.WHAT_A_NODE_IS_CALLED + " on the "+ settings.WHAT_A_GRAPH_IS_CALLED + "." err = "
"+ str(err) + "
" return HttpResponse(json.dumps({"result":"E", "message":err}), status=200) except nx.exception.NetworkXNoPath as err: #override err err = "
No path found between "+ settings.PATH_ORIGIN_LABEL +" and "+ settings.PATH_END_LABEL +"

NOTE: "+ settings.WHAT_A_CLUSTER_IS_CALLED +" can't be part of a path, but "+ settings.WHAT_A_NODE_IS_CALLED +" within a "+ settings.WHAT_A_CLUSTER_IS_CALLED +" can.
" return HttpResponse(json.dumps({"result":"E", "message":err}), status=200) except Warning as err: # How messages are returned if a change message transaction is expected: # FIXME: encapsulate in a def err = "
"+ str(err) + "
" return HttpResponse(json.dumps({"result":"E", "message":err}), status=200) except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) except Exception as err: return HttpResponse(str(err), status=406) analyze.alters_data = True # Limitation: Delete a tunnel in the origin map that has edges to its other end in the target map. Undo in the origin map will bring back the # tunnel in the target map, but won't bring back the edges in the target map. # FIXME: Edges won't un-select (Doesn't work in PHP site, either. Chrome only.) # FIXME: Can't turn tooltips off. (Doesn't work in PHP site, either.) @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() def commentSummary(request): # Build a view of all comments on all nodes in this map. # # @author Ron Newman # try: mapId = currently_opened_map(request) comments = NodeComment.objects.filter(node__ofmap=mapId,status=settings.NODE_COMMENT_ACTIVE).order_by('node').order_by('date') context = {'comments':comments, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED} return render(request,'ideatree/nodeCommentSummary.html',context) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) commentSummary.alters_data = True @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() def mapSummary(request): # Build a view of maps and who owns them try: update_mymap_list(request) maps = json.loads(request.session["accessible_maps"]) # FIXME: if map owner name needed elsewhere, put this in session["accessible_maps"] for amap in maps: amap["ownername"] = User.objects.get(pk=amap["owner"]).username return render(request,'ideatree/mapSummary.html', { 'maps':maps, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED }) except Exception as err: return HttpResponse(str(err), status=406) mapSummary.alters_data = True @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() @transaction.atomic() def vote(request): try: mapId = currently_opened_map(request) if request.session.get("readOnly"): raise Warning("Sorry, you are viewing in read-only mode, so you can't vote.") # FIXME: if read-only maps re-instated, do this with checkPermissions() votingUser = User.objects.get(pk=int(request.session["_auth_user_id"])) form = VoteForm(request.POST) if form.is_valid(): votableNodeId = form.cleaned_data["node"] voteLevel = form.cleaned_data["vote"] # FIXME: use actual node edgeect cross-checked with mapId for security. Vote.objects.update_or_create(user=votingUser, node=votableNodeId , defaults={'vote':voteLevel}) return makeResponse(status=600, body="Thank you for your vote!") else: return makeResponse(status=406, body="Internal error: invalid vote format.") except Warning as err: status=600 return makeResponse(status, str(err)) except Exception as err: return HttpResponse(str(err), status=406) vote.alters_data = True @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() def tallyVotes(request): try: mapId = currently_opened_map(request) # FIXME: abstract these preliminary checks out as a decoration? if("readOnly" in request.session): raise Warning("Sorry, you are viewing in read-only mode, so you can't vote.") if not str(mapId) in json.dumps(request.session["owned_maps"]): # actually, the UI should prevent ever getting to this raise Warning("Sorry, this function is reserved for the original creator of this "+settings.WHAT_A_GRAPH_IS_CALLED+".") votes = Vote.objects.filter(node__ofmap=mapId, status=settings.VOTABLE_ACTIVE) totalVotes = votes.count() votes = votes.values('node__label','vote') return render(request,'ideatree/tallyvotes.html', {'votes':votes, 'totalVotes':totalVotes, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED } ) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) # FIXME: deprecated @user_passes_test(not_guest_user) @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() # FIXME: obsolete def mapAccessTable(request): # Display a graph showing which users have access to this map, with the ability to deny/allow them. try: mapId = currently_opened_map(request) # Something from the table has been clicked, so make changes and re-render. # They should own this map. FIXME: test # If changing a map access, make sure the friend passed is in their friend list. userId = int(request.session["_auth_user_id"]) if 'f_id' in request.POST["f_id"]: friend = request.POST["f_id"] #numFriends = Friend.objects.filter(status not None, pk=friend, user=userId) # FIXME: is friend__status ambiguous? friends = Friend.objects.filter(status=settings.FRIEND_ACCEPTED, initiator=userId, friend__status=settings.USER_ACTIVE).exclude(friend__accounttype=settings.FREE_ACCT).exclude(friend__accounttype=settings.READONLY_ACCT) if 'allow' in request.POST: pass # do_query("insert into mapmembership (user_id,status,map_id) VALUES (:user_id,:status,:map_id)",array("user_id"=>$allow,"status"=>'V',"map_id"=>$mapID),__FILE__,__LINE__); // status='V'iew, 'H'ide that map for that user # set_map_access_changed($allow); if 'deny' in request.POST: pass #array("user_id"=>$deny,"map_id"=>$mapID); #do_query("delete from mapmembership where user_id=:user_id and map_id=:map_id",$vals,__FILE__,__LINE__); #set_map_access_changed($deny); # get the users assigned to this map # FIXME: most efficient? a way to do this without 'distinct'? why is distinct needed? # left join is used because we want both friends who do and who do not have access to this map # NOTE: important not to list readOnly users, though they be friends. This would open a security hole if they altered the URL map hash from #RO123 to #123. #"select distinct friends.friend_id, friends.status, mapmembership.user_id, users.username from #users, friends LEFT JOIN mapmembership #ON mapmembership.user_id=friends.friend_id and mapmembership.map_id=$mapID where friends.friend_id=users.ID #AND friends.status!='".DECLINED."' AND users.active='A' AND users.AccountType!='".READONLY_ACCT."' #AND friends.user_id=".$_SESSION["userID"]." order by friends.friend_id"; #mapmembers = Mapmember.objects.filter(status=settings.MAPMEMBERSHIP_ACTIVE, ofmap=mapId, member.initiator=userId, member__friend.accounttype!=settings.READONLY_ACCT, member__friend__user.status=settings.USER_ACTIVE, member__friend__status!=settings.FRIEND_DECLINED) #return render(request,'ideatree/mapAccessTable.html', context) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) mapAccessTable.alters_data = True @user_passes_test(not_guest_user) @transaction.atomic() @login_required() @csrf_protect def searchUsers(request): try: mapId = currently_opened_map(request) # Just to make sure there's a map open. if not 'doSearch' in request.POST: form = SearchAllUsersForm() else: form = SearchAllUsersForm(request.POST) if form.is_valid(): email = (form.cleaned_data['email'] ,) if any(email): #msg = FIXME: read from db. msg = "Come join us." fromAddr = settings.DEFAULT_FROM_EMAIL subject="Invitation To Collaborate From A Colleague" # FIXME For safety send msg, fromAddr to another form class for validation. try: send_mail(subject, msg, fromAddr, email, fail_silently=False) context['prompt'] = "Your invitation has been sent successfully!" thisUser = User.objects.get(pk=int(request.session["_auth_user_id"])) # Goes ahead and adds the invited user immediately as a friend, whether or not they sign up for an account. Friend.objects.update_or_create(initiator=thisUser) except BadHeaderError: return HttpResponse('Invalid header found.') except SMTPException as err: context['prompt'] = "Error while sending the invitation: " + str(err.reason) else: searchTerm = form.cleaned_data['searchTerm'] searchType = form.cleaned_data['searchType'] kwargs = {'{0}__{1}'.format(searchType,'istartswith'): str(searchTerm) } if not settings.GUEST_USERNAME: raise Exception("Missing guest username for exclusion.") foundUsers = User.objects.filter(**kwargs).exclude(pk=int(request.session["_auth_user_id"])).exclude(username=settings.GUEST_USERNAME) numUsersFound = foundUsers.count() # for test purposes table = UserSelectTable(foundUsers) RequestConfig(request).configure(table) context = {'numUsersFound':numUsersFound, 'table':table, 'title':'Search Results', 'navbuttontext':'Search Again'} return render(request, "ideatree/searchUserResults.html", context ) alphabetList = [ (alphaOrd,chr(alphaOrd)) for alphaOrd in range(ord('A'),ord('Z')+1)] context = { 'form':form, 'title':'Search Users', 'alphabetList':alphabetList } return render(request,'ideatree/searchUsers.html', context) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) searchUsers.alters_data = True @user_passes_test(not_guest_user) @login_required() @ensure_csrf_cookie # Because the form/render combination is not used. @transaction.atomic() def shareMap(request): try: thisUserId = int(request.session["_auth_user_id"]) sortby = request.POST['?sort'] if '?sort' in request.POST else None # FIXME: do this sort with django-tables2 API # FIXME: # friendQuery = "SELECT DISTINCT ideatree_friend.id, auth_user.username, first_name, last_name, auth_user.email, ideatree_map_desc.mapname, ideatree_map_desc.id as mapid\ # FROM ideatree_friend\ # LEFT JOIN auth_user ON auth_user.id=ideatree_friend.friend_id AND ideatree_friend.status !='"+ settings.FRIEND_DELETED + "' AND ideatree_friend.initiator_id = " + thisUser +"\ # LEFT JOIN ideatree_mapmember ON ideatree_mapmember.member_id=ideatree_friend.friend_id AND ideatree_mapmember.status !='"+ settings.MAPMEMBERSHIP_DELETED + "' \ # LEFT JOIN ideatree_map_desc ON ideatree_map_desc.id=ideatree_mapmember.ofmap_id AND ideatree_map_desc.owner_id="+ thisUser +" \ # WHERE auth_user.id !="+ thisUser + " AND mapname IS NOT NULL" sharedWith = [] # FIXME: is this the most efficient way? Not with prefetch_related() or chained select_related() or something? # The problem is the ORM has lots of things for the WHERE clause, but what for the SELECT clause, to vacuum up values? myfriends = Friend.objects.filter(initiator_id=thisUserId, status=settings.FRIEND_ACCEPTED) for fr in myfriends: mm = Mapmember.objects.filter(member=fr.friend_id, ofmap__owner_id=thisUserId, status=settings.MAPMEMBERSHIP_ACTIVE).select_related('ofmap','member') #mm = Mapmember.objects.filter(member__friend__initiator_id=thisUserId, member__friend__status=settings.FRIEND_ACCEPTED, ofmap__owner_id=thisUserId, status=settings.MAPMEMBERSHIP_ACTIVE).exclude(member_id=thisUserId).select_related('ofmap') for m in mm: # member in Mapmembers friend = {} friend["mapname"]=m.ofmap.mapname friend["mapid"]=m.ofmap.id # FIXME needed? friend["friend_id"]=m.member.id friend["username"]=m.member.username friend["first_name"]=m.member.first_name friend["last_name"]=m.member.last_name sharedWith.append(friend) if sortby: sharedWith = sorted(sharedWith, key=lambda k: k['username']) #friendQuery += " ORDER BY "+sortby+" ASC" #friends = Friend.objects.raw(friendQuery) table = FriendTable(sharedWith) RequestConfig(request).configure(table) context = {'table':table, 'title':settings.WHAT_A_GRAPH_IS_CALLED.capitalize()+'s You\'ve Shared', 'navbuttontext':"Share Current "+settings.WHAT_A_GRAPH_IS_CALLED.capitalize()+"..."} return render(request, "ideatree/searchUserResults.html", context ) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except FieldError as err: return HttpResponse(str(err), status=406) except Exception as err: return HttpResponse(str(err), status=406) shareMap.alters_data = True @user_passes_test(not_guest_user) @ensure_csrf_cookie @login_required() @transaction.atomic() @csrf_protect def addUserToMap(request): try: mapId = currently_opened_map(request) myUserId = int(request.session['_auth_user_id']) have_map_access(request, mapId) instance = Friend.objects.filter(initiator_id=myUserId).first() form = FriendDefineForm(request.POST, instance=instance) if form.is_valid(): newFriendId = form.cleaned_data['friend'].id Friend.objects.update_or_create(initiator_id=myUserId, friend_id=newFriendId, status=settings.FRIEND_ACCEPTED) Mapmember.objects.create(member_id=newFriendId, ofmap_id=mapId) return shareMap(request) else: raise Exception(form.errors) except Exception as err: return HttpResponse(str(err), status=406) except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) addUserToMap.alters_data = True @user_passes_test(not_guest_user) @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() @transaction.atomic() @csrf_protect def removeFromMap(request): try: # FIXME: form not used because input comes from client, but not from user-inputted form. ok? mapId = int(request.POST['mapId']) userId = int(request.POST['userId']) have_map_access(request, mapId) Mapmember.objects.filter(member_id=userId, ofmap_id=mapId).update(status=settings.MAPMEMBERSHIP_DELETED) return shareMap(request) except Exception as err: return HttpResponse(str(err), status=406) except PermissionDenied as err: status=600 body=str(err) return makeResponse(status, body) removeFromMap.alters_data = True @user_passes_test(not_guest_user) @ensure_csrf_cookie # Because the form/render combination is not used. @login_required() @transaction.atomic() @csrf_protect def removeFromGroup(request): try: thisUser = int(request.session["_auth_user_id"]) teamMember = int(request.POST['teamMemberId']) friends = friends_of(thisUser) if any(fr['friend_id']==int(teamMember) for fr in friends): Friend.objects.filter(initiator=thisUser, friend=teamMember).update(status=settings.FRIEND_DELETED) return manageTeam(request) return HttpResponse("Internal Error: this user is not a friend.", status=403) except Exception as err: return HttpResponse(str(err), status=406) removeFromGroup.alters_data = True @user_passes_test(not_guest_user) @login_required() @csrf_protect def notify(request, showUI=True, regarding=None): try: # FIXME write test of empty emailbody should return a template with form.error mapId = currently_opened_map(request) have_map_access(request, mapId) # a double check prompt = "" thisUserId = int(request.session["_auth_user_id"]) context = {'thisMapName':request.session["mapname"]} queryset = Mapmember.objects.filter(ofmap=mapId, status=settings.MAPMEMBERSHIP_ACTIVE).exclude(member_id=thisUserId) if queryset: context.update({'mapHasSharers':True}) # Write test if it's False. if not request.POST.get('submitted'): form = NotifyUsersForm(queryset=queryset, clear=False, emailbody_required=False) # NOTE: form will be invalid at this point because not bound. That's ok, we just want to fill out the template. else: form = NotifyUsersForm(request.POST, clear=False, queryset=queryset, emailbody_required=True) if form.is_valid(): prompt = "Your message has been sent successfully!" fromAddr = settings.DEFAULT_FROM_EMAIL subject="Notice From A Fellow IdeaTree User" sender = User.objects.get(pk=request.session["_auth_user_id"]) msg="From: " + sender.username + "\n" if regarding: msg += "Re: " + regarding + "\n" msg += "\nMessage:\n" + form.cleaned_data['emailbody'] emails = [] for selectedUser in form.cleaned_data['users']: if selectedUser.member.email: # filter out empty strings so that the following check for empty list works emails.append(selectedUser.member.email) if not emails: # FIXME: list specific emails that are missing. Write test and also test in UI. raise Warning("No email address found for selected user(s).") try: # FIXME: test when available send_mail(subject, msg, fromAddr, emails, fail_silently=False,) # get ready for a fresh start form = NotifyUsersForm(queryset=queryset, clear=True, emailbody_required=False) # FIXME: does send_mail send Exceptions? except Exception as err: if showUI: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") else: # FIXME test this coming from node comment return makeResponse(status=600, body=str(err)) # For responses to Ajax calls, like from node comments. if showUI: context.update({"form":form, 'prompt':prompt, 'whatagraphiscalled':settings.WHAT_A_GRAPH_IS_CALLED }) return render(request, 'ideatree/notify.html', context) except Warning as err: context = {'error':str(err)} return render(request, "showWarning.html", context, content_type="application/html") except Exception as err: return HttpResponse(str(err), status=406) notify.alters_data = True @ensure_csrf_cookie @login_required() @csrf_protect def saveMapSettings(request): try: userId = int(request.session["_auth_user_id"]) form = MapSettingsForm(request.POST) if not form.is_valid(): raise Exception("Internal error: Invalid data sent.") new_mapsettings = json.loads(form.cleaned_data.get("mapsettings")) if not new_mapsettings: # reverting to the db defaults # FIXME: use setattr instead, so that the constant doesn't have to be remembered. UserProfile.objects.filter(user__id=userId).update(mapsettings=settings.USER_DEFAULT_MAP_SETTINGS) else: mapsettings = json.loads(UserProfile.objects.get(user__id=userId).mapsettings) for key,val in new_mapsettings.items(): mapsettings[key] = val UserProfile.objects.filter(user__id=userId).update(mapsettings=json.dumps(mapsettings)) return makeResponse(200,"") except UserProfile.DoesNotExist: raise Exception("User profile does not exist for id:"+ str(userId)) except Exception as err: return HttpResponse(str(err), status=406) saveMapSettings.alters_data = True @ensure_csrf_cookie @login_required() def whosLoggedIn(request): # Real-time list of who is viewing the current map (doesn't apply to embedded, read-only maps) try: mapId = currently_opened_map(request) vList = [] viewers = WhosLoggedIn.objects.select_related('user').filter(currentmap_id=mapId) for v in viewers: vList.append(v.user.username) return makeResponse(status=200, body={"message":vList}) except Exception as err: return HttpResponse(str(err), status=406) # FIXME: move these two to utils/myutils.py and which of these is used? # Taken from: http://docs.python-guide.org/en/latest/scenarios/scrape/ def get_remote_title(url): import requests from lxml import html try: page = requests.get(url) tree = html.fromstring(page.content) title = tree.xpath('/html/head/title') if len(title): title = title[0].text return title; else: return "" except requests.RequestException as err: raise Warning("Looks like that site may have refused connection.") @ensure_csrf_cookie @login_required() def getRemoteSiteTitle(request): try: import validators if request.POST.get("remoteURI"): url=request.POST["remoteURI"] else: return makeResponse(status=406, body={"message":"NO_URI_SUPPLIED"}) # validate the url first. validators.url(url) # and extra validation for security if not re.match(settings.URL_SAFETY_REGEX,url): raise Warning("Sorry, that site's link did not pass the safety test,
so it can't be used to create a linked " + settings.WHAT_A_NODE_IS_CALLED + ".

Always looking out for you.") # scrape the title from the remote location and validate it title = get_remote_title(url) if title: if not re.match(settings.STRING_SAFETY_REGEX,title): title = "" return makeResponse(status=200, body={"message":title}) except Warning as err: return makeResponse(status=600, body=str(err)) except Exception as err: return HttpResponse(str(err), status=406)