Merge pull request #1622 from willimoa/enhancement/d3_graph

Enhancement/d3 graph

applications/admin/controllers/appadmin.py

@@ -12,6 +12,7 @@ import gluon.contenttype
 import gluon.fileutils
 from gluon._compat import iteritems
 
+# d3_graph_model added but leaving pygraphviz code as is for initial tests.
 try:
     import pygraphviz as pgv
 except ImportError:
@@ -566,6 +567,9 @@ def table_template(table):
                              ).xml()
 
 
+# d3_graph_model added but leaving pygraphviz code as is for initial tests.
+# The Graph Model button in admin app  views/default/design.html has been redirected 
+# to the d3_graph_model function.
 def bg_graph_model():
     graph = pgv.AGraph(layout='dot',  directed=True,  strict=False,  rankdir='LR')
 
@@ -700,3 +704,52 @@ def hooks():
             ul_t.append(UL([LI(A(f['funcname'], _class="editor_filelink", _href=f['url']if 'url' in f else None, **{'_data-lineno':f['lineno']-1})) for f in op['functions']]))
         ul_main.append(ul_t)
     return ul_main
+
+
+# ##########################################################
+# d3 based model visualizations
+# ###########################################################
+
+def d3_graph_model():
+    """ See https://www.facebook.com/web2py/posts/145613995589010 from Bruno Rocha
+    and also the app_admin bg_graph_model function
+    
+    Create a list of table dicts, called "nodes"
+    """
+    
+    data = {}
+    nodes = []
+    links = []
+
+    subgraphs = dict()
+
+    for tablename in db.tables:
+        fields = []
+        for field in db[tablename]:
+            f_type = field.type
+            if not isinstance(f_type,str):
+                disp = ' '
+            elif f_type == 'string':
+                disp =  field.length
+            elif f_type == 'id':
+                disp =  "PK"
+            elif f_type.startswith('reference') or \
+                f_type.startswith('list:reference'):
+                disp = "FK"
+            else:
+                disp = ' '
+            fields.append(dict(name= field.name, type=field.type, disp = disp))
+
+            if isinstance(f_type,str) and (
+                f_type.startswith('reference') or
+                f_type.startswith('list:reference')):
+                referenced_table = f_type.split()[1].split('.')[0]
+
+                links.append(dict(source=tablename, target = referenced_table))
+
+        nodes.append(dict(name=tablename, type="table", fields = fields))
+
+    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
+    response.files.append(URL('static','js/d3.min.js'))
+    response.files.append(URL('static','js/d3_graph.js'))
+    return dict(nodes=nodes, links=links)

applications/admin/views/appadmin.html

@@ -258,6 +258,19 @@
   {{pass}}
 {{pass}}
 
+{{if request.function=='d3_graph_model':}}
+<h2>{{=T("Graph Model")}}</h2>
+<div id="vis"></div>
+<link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+<script>
+    // Define the d3 input data
+    {{from gluon.serializers import json }}
+    var nodes = {{=XML(json(nodes))}};
+    var links = {{=XML(json(links))}};
+    d3_graph();
+</script>
+{{pass}}
+
 {{if request.function == 'manage':}}
 <h2>{{=heading}}</h2>
 <ul class="nav nav-tabs">

applications/admin/views/default/design.html

@@ -105,7 +105,11 @@ def deletefile(arglist, vars={}):
         {{if os.access(os.path.join(request.folder,'..',app,'databases','sql.log'),os.R_OK):}}
             {{=button(URL('peek/%s/databases/sql.log'%app), 'sql.log')}}
         {{pass}}
+        {{if os.access(os.path.join(request.folder,'..',app,'static','js','d3_graph.js'),os.R_OK):}}
+                {{=button(URL(a=app, c='appadmin',f='d3_graph_model'), T('graph model'))}}
+        {{else:}}
                 {{=button(URL(a=app, c='appadmin',f='graph_model'), T('graph model'))}}
+        {{pass}}
     </div>
     <ul class="unstyled act_edit">
         {{for m in models:}}

applications/examples/controllers/appadmin.py

@@ -12,6 +12,7 @@ import gluon.contenttype
 import gluon.fileutils
 from gluon._compat import iteritems
 
+# d3_graph_model added but leaving pygraphviz code as is for initial tests.
 try:
     import pygraphviz as pgv
 except ImportError:
@@ -566,6 +567,9 @@ def table_template(table):
                              ).xml()
 
 
+# d3_graph_model added but leaving pygraphviz code as is for initial tests.
+# The Graph Model button in admin app  views/default/design.html has been redirected 
+# to the d3_graph_model function.
 def bg_graph_model():
     graph = pgv.AGraph(layout='dot',  directed=True,  strict=False,  rankdir='LR')
 
@@ -700,3 +704,52 @@ def hooks():
             ul_t.append(UL([LI(A(f['funcname'], _class="editor_filelink", _href=f['url']if 'url' in f else None, **{'_data-lineno':f['lineno']-1})) for f in op['functions']]))
         ul_main.append(ul_t)
     return ul_main
+
+
+# ##########################################################
+# d3 based model visualizations
+# ###########################################################
+
+def d3_graph_model():
+    """ See https://www.facebook.com/web2py/posts/145613995589010 from Bruno Rocha
+    and also the app_admin bg_graph_model function
+    
+    Create a list of table dicts, called "nodes"
+    """
+    
+    data = {}
+    nodes = []
+    links = []
+
+    subgraphs = dict()
+
+    for tablename in db.tables:
+        fields = []
+        for field in db[tablename]:
+            f_type = field.type
+            if not isinstance(f_type,str):
+                disp = ' '
+            elif f_type == 'string':
+                disp =  field.length
+            elif f_type == 'id':
+                disp =  "PK"
+            elif f_type.startswith('reference') or \
+                f_type.startswith('list:reference'):
+                disp = "FK"
+            else:
+                disp = ' '
+            fields.append(dict(name= field.name, type=field.type, disp = disp))
+
+            if isinstance(f_type,str) and (
+                f_type.startswith('reference') or
+                f_type.startswith('list:reference')):
+                referenced_table = f_type.split()[1].split('.')[0]
+
+                links.append(dict(source=tablename, target = referenced_table))
+
+        nodes.append(dict(name=tablename, type="table", fields = fields))
+
+    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
+    response.files.append(URL('static','js/d3.min.js'))
+    response.files.append(URL('static','js/d3_graph.js'))
+    return dict(nodes=nodes, links=links)

applications/examples/views/appadmin.html

@@ -258,6 +258,19 @@
   {{pass}}
 {{pass}}
 
+{{if request.function=='d3_graph_model':}}
+<h2>{{=T("Graph Model")}}</h2>
+<div id="vis"></div>
+<link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+<script>
+    // Define the d3 input data
+    {{from gluon.serializers import json }}
+    var nodes = {{=XML(json(nodes))}};
+    var links = {{=XML(json(links))}};
+    d3_graph();
+</script>
+{{pass}}
+
 {{if request.function == 'manage':}}
 <h2>{{=heading}}</h2>
 <ul class="nav nav-tabs">

applications/welcome/controllers/appadmin.py

@@ -12,6 +12,7 @@ import gluon.contenttype
 import gluon.fileutils
 from gluon._compat import iteritems
 
+# d3_graph_model added but leaving pygraphviz code as is for initial tests.
 try:
     import pygraphviz as pgv
 except ImportError:
@@ -566,6 +567,9 @@ def table_template(table):
                              ).xml()
 
 
+# d3_graph_model added but leaving pygraphviz code as is for initial tests.
+# The Graph Model button in admin app  views/default/design.html has been redirected 
+# to the d3_graph_model function.
 def bg_graph_model():
     graph = pgv.AGraph(layout='dot',  directed=True,  strict=False,  rankdir='LR')
 
@@ -700,3 +704,52 @@ def hooks():
             ul_t.append(UL([LI(A(f['funcname'], _class="editor_filelink", _href=f['url']if 'url' in f else None, **{'_data-lineno':f['lineno']-1})) for f in op['functions']]))
         ul_main.append(ul_t)
     return ul_main
+
+
+# ##########################################################
+# d3 based model visualizations
+# ###########################################################
+
+def d3_graph_model():
+    """ See https://www.facebook.com/web2py/posts/145613995589010 from Bruno Rocha
+    and also the app_admin bg_graph_model function
+    
+    Create a list of table dicts, called "nodes"
+    """
+    
+    data = {}
+    nodes = []
+    links = []
+
+    subgraphs = dict()
+
+    for tablename in db.tables:
+        fields = []
+        for field in db[tablename]:
+            f_type = field.type
+            if not isinstance(f_type,str):
+                disp = ' '
+            elif f_type == 'string':
+                disp =  field.length
+            elif f_type == 'id':
+                disp =  "PK"
+            elif f_type.startswith('reference') or \
+                f_type.startswith('list:reference'):
+                disp = "FK"
+            else:
+                disp = ' '
+            fields.append(dict(name= field.name, type=field.type, disp = disp))
+
+            if isinstance(f_type,str) and (
+                f_type.startswith('reference') or
+                f_type.startswith('list:reference')):
+                referenced_table = f_type.split()[1].split('.')[0]
+
+                links.append(dict(source=tablename, target = referenced_table))
+
+        nodes.append(dict(name=tablename, type="table", fields = fields))
+
+    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
+    response.files.append(URL('static','js/d3.min.js'))
+    response.files.append(URL('static','js/d3_graph.js'))
+    return dict(nodes=nodes, links=links)

applications/welcome/static/css/d3_graph.css

@@ -0,0 +1,33 @@
+    .node {fill: steelblue;
+           stroke: #636363;
+           stroke-width: 1px;}
+
+    .auth {fill: lightgrey;}
+
+    .table {r: 10;}
+
+    .link {stroke: #bbbbbb;
+           stroke-width: 2px;}
+    td {padding: 4px;}
+
+    div.tooltip {
+       position: absolute;
+       text-align: left;
+       /* width: 140px; */
+       /* height: 28px;*/
+       padding: 0px 5px 0px 5px;
+       padding-top: 0px;
+       font: 12px sans-serif;
+       background: #fff7bc;
+       border: solid 1px #aaa;
+       border-radius: 6px;
+       pointer-events: none;}
+
+    h5 { font: 14px sans-serif;
+         background : #ec7014;
+         color: #ffffe5; 
+         padding: 5px 2px 5px 2px;
+         margin-top: 1px;}
+    path {
+           fill: #aaaaaa;}
+

applications/welcome/static/js/d3_graph.js

@@ -0,0 +1,181 @@
+function d3_graph() {
+
+// Some reference links:
+//   How to get link ids instead of index
+//     http://stackoverflow.com/questions/23986466/d3-force-layout-linking-nodes-by-name-instead-of-index
+//   embedding web2py in d3
+//     http://stackoverflow.com/questions/34326343/embedding-d3-js-graph-in-a-web2py-bootstrap-page
+
+// nodes and links are defined in appadmin.html <script>
+
+
+    var edges = [];
+
+    links.forEach(function(e) {
+        var sourceNode = nodes.filter(function(n) {
+            return n.name === e.source;
+        })[0],
+            targetNode = nodes.filter(function(n) {
+            return n.name === e.target;
+        })[0];
+
+        edges.push({
+            source: sourceNode,
+            target: targetNode,
+            value: 1});
+
+    });
+
+    edges.forEach(function(e) {
+
+        if (!e.source["linkcount"]) e.source["linkcount"] = 0;
+        if (!e.target["linkcount"]) e.target["linkcount"] = 0;
+
+        e.source["linkcount"]++;
+        e.target["linkcount"]++;
+    });
+
+    //var width = 960, height = 600;
+    var height = window.innerHeight|| docEl.clientHeight|| bodyEl.clientHeight;
+    var width = window.innerWidth || docEl.clientWidth || bodyEl.clientWidth;
+    var svg = d3.select("#vis").append("svg")
+            .attr("width", width)
+            .attr("height", height);
+
+       // updated for d3 v4.
+    var simulation = d3.forceSimulation()
+            .force("link", d3.forceLink().id(function(d) { return d.id; }))
+            .force("charge", d3.forceManyBody().strength(strength))
+            .force("center", d3.forceCenter(width / 2, height / 2))
+            .force("collision", d3.forceCollide(35));
+
+    // Node charge strength.  Repel strength greater for less links.
+    //function strength(d) { return -50/d["linkcount"] ; }
+    function strength(d) { return -25 ; }  
+    
+    // Link distance.  Distance increases with number of links at source and target
+    function distance(d) { return (60 + (d.source["linkcount"] * d.target["linkcount"])) ; }
+    
+    // Link strength.  Strength is less for highly connected nodes (move towards target dist)
+    function strengthl(d) { return 5/(d.source["linkcount"] + d.target["linkcount"]) ; }
+
+    simulation
+        .nodes(nodes)
+        .on("tick", tick);
+
+    simulation.force("link")
+        .links(edges)
+        .distance(distance)
+        .strength(strengthl);
+
+    // build the arrow.
+    svg.append("svg:defs").selectAll("marker")
+        .data(["end"])      // Different link/path types can be defined here
+        .enter().append("svg:marker")    // This section adds in the arrows
+        .attr("id", String)
+        .attr("viewBox", "0 -5 10 10")
+        .attr("refX", 25)   // Moves the arrow head out, allow for radius
+        .attr("refY", 0)   // -1.5
+        .attr("markerWidth", 6)
+        .attr("markerHeight", 6)
+        .attr("orient", "auto")
+        .append("svg:path")
+        .attr("d", "M0,-5L10,0L0,5");
+
+    var link = svg.selectAll('.link')
+        .data(edges)
+        .enter().append('line')
+        .attr("class", "link")
+        .attr("marker-end", "url(#end)");
+
+    var node = svg.selectAll(".node")
+        .data(nodes)
+        .enter().append("g")
+        .attr("class", function(d) { return "node " + d.type;})
+        .attr('transform', function(d) {
+            return "translate(" + d.x + "," + d.y + ")"})
+        .classed("auth", function(d) { return (d.name.startsWith("auth") ? true : false);});
+
+    node.call(d3.drag()
+            .on("start", dragstarted)
+            .on("drag", dragged)
+            .on("end", dragended));
+
+    // add the nodes
+    node.append('circle')
+        .attr('r', 16)
+        ;
+
+    // add text
+    node.append("text")
+        .attr("x", 12)
+        .attr("dy", "-1.1em")
+        .text(function(d) {return d.name;});
+
+    node.on("mouseover", function(d) {
+
+        var g = d3.select(this);  // the node (table)
+
+        // tooltip
+
+        var fields = d.fields;
+        var fieldformat = "<TABLE>";
+        fields.forEach(function(d) {
+            fieldformat += "<TR><TD><B>"+ d.name+"</B></TD><TD>"+ d.type+"</TD><TD>"+ d.disp+"</TD></TR>";
+        });
+        fieldformat += "</TABLE>";
+        var tiplength = d.fields.length;
+
+        // Define 'div' for tooltips
+        var div = d3.select("body").append("div")  // declare the tooltip div
+	        .attr("class", "tooltip")              // apply the 'tooltip' class
+            .style("opacity", 0)
+            .html('<h5>' + d.name + '</h5>' + fieldformat)
+            .style("left", 20 + (d3.event.pageX) + "px")// or just (d.x + 50 + "px")
+            .style("top", tooltop(tiplength))// or ...
+            .transition()
+            .duration(800)
+            .style("opacity", 0.9);
+        });
+
+        function tooltop(tiplength) {
+           //aim to ensure tooltip is fully visible whenver possible
+           return (Math.max(d3.event.pageY - 20 - (tiplength * 14),0)) + "px"
+        }
+
+        node.on("mouseout", function(d) {
+            d3.select("body").select('div.tooltip').remove();
+    });
+
+    // instead of waiting for force to end with :     force.on('end', function()
+    // use .on("tick",  instead.  Here is the tick function
+    function tick() {
+        node.attr('transform', function(d) {
+            d.x = Math.max(30, Math.min(width - 16, d.x));
+            d.y = Math.max(30, Math.min(height - 16, d.y));
+            return "translate(" + d.x + "," + d.y + ")"; });
+
+        link.attr('x1', function(d) {return d.source.x;})
+            .attr('y1', function(d) {return d.source.y;})
+            .attr('x2', function(d) {return d.target.x;})
+            .attr('y2', function(d) {return d.target.y;});
+    };
+
+    function dragstarted(d) {
+        if (!d3.event.active) simulation.alphaTarget(0.3).restart();
+        d.fx = d.x;
+        d.fy = d.y;
+    };
+
+    function dragged(d) {
+        d.fx = d3.event.x;
+        d.fy = d3.event.y;
+    };
+
+    function dragended(d) {
+        if (!d3.event.active) simulation.alphaTarget(0);
+        d.fx = null;
+        d.fy = null;
+    };
+
+};

applications/welcome/views/appadmin.html

@@ -258,6 +258,19 @@
   {{pass}}
 {{pass}}
 
+{{if request.function=='d3_graph_model':}}
+<h2>{{=T("Graph Model")}}</h2>
+<div id="vis"></div>
+<link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+<script>
+    // Define the d3 input data
+    {{from gluon.serializers import json }}
+    var nodes = {{=XML(json(nodes))}};
+    var links = {{=XML(json(links))}};
+    d3_graph();
+</script>
+{{pass}}
+
 {{if request.function == 'manage':}}
 <h2>{{=heading}}</h2>
 <ul class="nav nav-tabs">