path: root/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py

#!/usr/bin/env python3
#
# Copyright 2020 MongoDB Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
"""
Flask backend web server.

The backend interacts with the graph_analyzer to perform queries on various libdeps graphs.
"""

from pathlib import Path
from collections import namedtuple, OrderedDict

import flask
import networkx

from flask_socketio import SocketIO, emit
from flask_cors import CORS
from flask_session import Session
from lxml import etree

import libdeps.graph
import libdeps.analyzer


class BackendServer:
    """Create small class for storing variables and state of the backend."""

    # pylint: disable=too-many-instance-attributes
    def __init__(self, graphml_dir, frontend_url):
        """Create and setup the state variables."""
        self.app = flask.Flask(__name__)
        self.socketio = SocketIO(self.app, cors_allowed_origins=frontend_url)
        self.app.config['CORS_HEADERS'] = 'Content-Type'
        CORS(self.app, resources={r"/*": {"origins": frontend_url}})

        self.app.add_url_rule("/graph_files", "return_graph_files", self.return_graph_files)
        self.socketio.on_event('git_hash_selected', self.git_hash_selected)
        self.socketio.on_event('row_selected', self.row_selected)

        self.loaded_graphs = {}
        self.current_selected_rows = {}
        self.graphml_dir = Path(graphml_dir)
        self.frontend_url = frontend_url

        self.graph_file_tuple = namedtuple('GraphFile', ['version', 'git_hash', 'graph_file'])
        self.graph_files = self.get_graphml_files()

        try:
            default_selected_graph = list(self.graph_files.items())[0][1].graph_file
            self.load_graph_from_file(default_selected_graph)
            self._dependents_graph = networkx.reverse_view(self._dependency_graph)
        except (IndexError, AttributeError) as ex:
            print(ex)
            print(
                f"Failed to load read a graph file from {list(self.graph_files.items())} for graphml_dir '{self.graphml_dir}'"
            )
            exit(1)

    def load_graph_from_file(self, file_path):
        """Load a graph file from disk and handle version."""

        graph = libdeps.graph.LibdepsGraph(networkx.read_graphml(file_path))
        if graph.graph['graph_schema_version'] == 1:
            self._dependents_graph = graph
            self._dependency_graph = networkx.reverse_view(self._dependents_graph)
        else:
            self._dependency_graph = graph
            self._dependents_graph = networkx.reverse_view(self._dependency_graph)

    def get_app(self):
        """Return the app and socketio instances."""

        return self.app, self.socketio

    def get_graph_build_data(self, graph_file):
        """Fast method for extracting basic build data from the graph file."""

        version = ''
        git_hash = ''
        # pylint: disable=c-extension-no-member
        for _, element in etree.iterparse(
                str(graph_file), tag="{http://graphml.graphdrawing.org/xmlns}data"):
            if element.get('key') == 'graph_schema_version':
                version = element.text
            if element.get('key') == 'git_hash':
                git_hash = element.text
            element.clear()
            if version and git_hash:
                break
        return self.graph_file_tuple(version, git_hash, graph_file)

    def get_graphml_files(self):
        """Find all graphml files in the target graphml dir."""

        graph_files = OrderedDict()
        for graph_file in self.graphml_dir.glob("**/*.graphml"):
            graph_file_tuple = self.get_graph_build_data(graph_file)
            graph_files[graph_file_tuple.git_hash[:7]] = graph_file_tuple
        return graph_files

    def return_graph_files(self):
        """Prepare the list of graph files for the frontend."""

        data = {'graph_files': []}
        for i, (_, graph_file_data) in enumerate(self.graph_files.items(), start=1):
            data['graph_files'].append({
                'id': i, 'version': graph_file_data.version, 'git': graph_file_data.git_hash[:7],
                'selected': False
            })
        return data

    def send_node_infos(self):
        """Search through the selected rows and find information about the selected rows."""

        with self.app.test_request_context():

            nodeinfo_data = {'nodeInfos': []}

            for node, _ in self.current_selected_rows.items():

                nodeinfo_data['nodeInfos'].append({
                    'id':
                        len(nodeinfo_data['nodeInfos']),
                    'node':
                        str(node),
                    'name':
                        node.name,
                    'attribs': [{
                        'name': key, 'value': value
                    } for key, value in self._dependents_graph.nodes(data=True)[str(node)].items()],
                    'dependers': [{
                        'node':
                            depender, 'symbols':
                                self._dependents_graph[str(node)][depender].get('symbols',
                                                                                '').split(' ')
                    } for depender in self._dependents_graph[str(node)]],
                    'dependencies': [{
                        'node':
                            dependency, 'symbols':
                                self._dependents_graph[dependency][str(node)].get('symbols',
                                                                                  '').split(' ')
                    } for dependency in self._dependency_graph[str(node)]],
                })

            self.socketio.emit("node_infos", nodeinfo_data)

    def send_graph_data(self):
        """Convert the current selected rows into a format for D3."""

        with self.app.test_request_context():

            nodes = set()
            links = set()

            for node, _ in self.current_selected_rows.items():
                nodes.add(
                    tuple({
                        'id': str(node), 'name': node.name, 'type': self._dependents_graph.nodes()
                                                                    [str(node)]['bin_type']
                    }.items()))

                for depender in self._dependency_graph[str(node)]:

                    depender_path = Path(depender)
                    if self._dependents_graph[depender][str(node)].get('direct'):
                        nodes.add(
                            tuple({
                                'id':
                                    str(depender_path), 'name':
                                        depender_path.name, 'type':
                                            self._dependents_graph.nodes()[str(depender_path)]
                                            ['bin_type']
                            }.items()))
                        links.add(
                            tuple({'source': str(node), 'target': str(depender_path)}.items()))

            node_data = {
                'graphData': {
                    'nodes': [dict(node) for node in nodes],
                    'links': [dict(link) for link in links],
                }, 'selectedNodes': [str(node) for node in list(self.current_selected_rows.keys())]
            }
            self.socketio.emit("graph_data", node_data)

    def row_selected(self, message):
        """Construct the new graphData nodeInfo when a cell is selected."""

        print(f"Got row {message}!")

        if message['isSelected'] == 'flip':
            if message['data']['node'] in self.current_selected_rows:
                self.current_selected_rows.pop(message['data']['node'])
            else:
                self.current_selected_rows[Path(message['data']['node'])] = message['data']
        else:
            if message['isSelected'] and message:
                self.current_selected_rows[Path(message['data']['node'])] = message['data']
            else:
                self.current_selected_rows.pop(message['data']['node'])

        self.socketio.start_background_task(self.send_graph_data)
        self.socketio.start_background_task(self.send_node_infos)

    def analyze_counts(self):
        """Perform count analysis and send the results back to frontend."""

        with self.app.test_request_context():

            analysis = libdeps.analyzer.counter_factory(
                self._dependents_graph,
                [name[0] for name in libdeps.analyzer.CountTypes.__members__.items()])
            ga = libdeps.analyzer.LibdepsGraphAnalysis(analysis)
            results = ga.get_results()

            graph_data = []
            for i, data in enumerate(results):
                graph_data.append({'id': i, 'type': data, 'value': results[data]})
            self.socketio.emit("graph_results", graph_data)

    def send_node_list(self):
        """Gather all the nodes in the graph for the node list."""

        with self.app.test_request_context():
            node_data = {
                'graphData': {'nodes': [], 'links': []},
                "selectedNodes": [str(node) for node in list(self.current_selected_rows.keys())]
            }

            for node in self._dependents_graph.nodes():
                node_path = Path(node)
                node_data['graphData']['nodes'].append(
                    {'id': str(node_path), 'name': node_path.name})
            self.socketio.emit("graph_nodes", node_data)

    def load_graph(self, message):
        """Load the graph into application memory and kick off threads for analysis on new graph."""

        with self.app.test_request_context():

            current_hash = self._dependents_graph.graph.get('git_hash', 'NO_HASH')[:7]
            if current_hash != message['hash']:
                self.current_selected_rows = {}
                if message['hash'] in self.loaded_graphs:
                    self._dependents_graph = self.loaded_graphs[message['hash']]
                    self._dependents_graph = networkx.reverse_view(self._dependency_graph)
                else:
                    print(
                        f'loading new graph {current_hash} because different than {message["hash"]}'
                    )

                    self.load_graph_from_file(self.graph_files[message['hash']].graph_file)
                    self.loaded_graphs[message['hash']] = self._dependents_graph

            self.socketio.start_background_task(self.analyze_counts)
            self.socketio.start_background_task(self.send_node_list)
            self.socketio.emit("graph_data", {'graphData': {'nodes': [], 'links': []}})

    def git_hash_selected(self, message):
        """Load the new graph and perform queries on it."""

        print(f"Got requests2 {message}!")

        emit("other_hash_selected", message, broadcast=True)

        self.socketio.start_background_task(self.load_graph, message)