SERVER-53531 added evergreen task for libdeps graph generation

3 files changed, 98 insertions, 35 deletions

diff --git a/buildscripts/libdeps/gacli.py b/buildscripts/libdeps/gacli.py
index 9121702c5bd..b01ec5b0d4f 100755
--- a/buildscripts/libdeps/gacli.py
+++ b/buildscripts/libdeps/gacli.py
@@ -34,7 +34,7 @@ from pathlib import Path
 
 import networkx
 import graph_analyzer
-
+from libdeps_graph_enums import CountTypes, LinterTypes
 
 class LinterSplitArgs(argparse.Action):
     """Custom argument action for checking multiple choice comma separated list."""
@@ -58,17 +58,13 @@ class LinterSplitArgs(argparse.Action):
 class CountSplitArgs(LinterSplitArgs):
     """Special case of common custom arg action for Count types."""
 
-    valid_choices = [
-        name[0].replace('_', '-') for name in graph_analyzer.CountTypes.__members__.items()
-    ]
+    valid_choices = [name[0].replace('_', '-') for name in CountTypes.__members__.items()]
 
 
 class LintSplitArgs(LinterSplitArgs):
     """Special case of common custom arg action for Count types."""
 
-    valid_choices = [
-        name[0].replace('_', '-') for name in graph_analyzer.LinterTypes.__members__.items()
-    ]
+    valid_choices = [name[0].replace('_', '-') for name in LinterTypes.__members__.items()]
 
 
 class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
@@ -85,30 +81,30 @@ class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHe
     def _get_help_string(self, action):
 
         if isinstance(action, CountSplitArgs):
-            count_help = self._get_help_length(graph_analyzer.CountTypes)
+            count_help = self._get_help_length(CountTypes)
             return textwrap.dedent(f"""\
                 {action.help}
                 default: all, choices:
-                    {count_help[graph_analyzer.CountTypes.all.name]}perform all counts
-                    {count_help[graph_analyzer.CountTypes.node.name]}count nodes
-                    {count_help[graph_analyzer.CountTypes.edge.name]}count edges
-                    {count_help[graph_analyzer.CountTypes.dir_edge.name]}count edges declared directly on a node
-                    {count_help[graph_analyzer.CountTypes.trans_edge.name]}count edges induced by direct public edges
-                    {count_help[graph_analyzer.CountTypes.dir_pub_edge.name]}count edges that are directly public
-                    {count_help[graph_analyzer.CountTypes.pub_edge.name]}count edges that are public
-                    {count_help[graph_analyzer.CountTypes.priv_edge.name]}count edges that are private
-                    {count_help[graph_analyzer.CountTypes.if_edge.name]}count edges that are interface
-                    {count_help[graph_analyzer.CountTypes.shim.name]}count shim nodes
-                    {count_help[graph_analyzer.CountTypes.lib.name]}count library nodes
-                    {count_help[graph_analyzer.CountTypes.prog.name]}count program nodes
+                    {count_help[CountTypes.all.name]}perform all counts
+                    {count_help[CountTypes.node.name]}count nodes
+                    {count_help[CountTypes.edge.name]}count edges
+                    {count_help[CountTypes.dir_edge.name]}count edges declared directly on a node
+                    {count_help[CountTypes.trans_edge.name]}count edges induced by direct public edges
+                    {count_help[CountTypes.dir_pub_edge.name]}count edges that are directly public
+                    {count_help[CountTypes.pub_edge.name]}count edges that are public
+                    {count_help[CountTypes.priv_edge.name]}count edges that are private
+                    {count_help[CountTypes.if_edge.name]}count edges that are interface
+                    {count_help[CountTypes.shim.name]}count shim nodes
+                    {count_help[CountTypes.lib.name]}count library nodes
+                    {count_help[CountTypes.prog.name]}count program nodes
                 """)
         elif isinstance(action, LintSplitArgs):
-            count_help = self._get_help_length(graph_analyzer.LinterTypes)
+            count_help = self._get_help_length(LinterTypes)
             return textwrap.dedent(f"""\
                 {action.help}
                 default: all, choices:
-                    {count_help[graph_analyzer.LinterTypes.all.name]}perform all linters
-                    {count_help[graph_analyzer.LinterTypes.node.name]}find unnecessary public libdeps
+                    {count_help[LinterTypes.all.name]}perform all linters
+                    {count_help[LinterTypes.node.name]}find unnecessary public libdeps
                 """)
         return super()._get_help_string(action)
 
@@ -124,23 +120,27 @@ def setup_args_parser():
     parser.add_argument('--format', choices=['pretty', 'json'], default='pretty',
                         help="The output format type.")
 
-    parser.add_argument('--build-data', action='store_true',
+    parser.add_argument('--build-data', choices=['on', 'off'], default='on',
                         help="Print the invocation and git hash used to build the graph")
 
-    parser.add_argument('--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs,
-                        help="Output various counts from the graph. Comma separated list.")
+    parser.add_argument(
+        '--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs,
+        default=[name[0] for name in CountTypes.__members__.items() if name[0] != 'all'],
+        help="Output various counts from the graph. Comma separated list.")
 
-    parser.add_argument('--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs,
-                        help="Perform various linters on the graph. Comma separated list.")
+    parser.add_argument(
+        '--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs,
+        default=[name[0] for name in LinterTypes.__members__.items() if name[0] != 'all'],
+        help="Perform various linters on the graph. Comma separated list.")
 
-    parser.add_argument('--direct-depends', action='append',
+    parser.add_argument('--direct-depends', action='append', default=[],
                         help="Print the nodes which depends on a given node.")
 
-    parser.add_argument('--common-depends', nargs='+', action='append',
+    parser.add_argument('--common-depends', nargs='+', action='append', default=[],
                         help="Print the nodes which have a common dependency on all N nodes.")
 
     parser.add_argument(
-        '--exclude-depends', nargs='+', action='append', help=
+        '--exclude-depends', nargs='+', action='append', default=[], help=
         "Print nodes which depend on the first node of N nodes, but exclude all nodes listed there after."
     )
 
diff --git a/buildscripts/libdeps/graph_analyzer.py b/buildscripts/libdeps/graph_analyzer.py
index 52defa1ab32..c9138b2dc63 100644
--- a/buildscripts/libdeps/graph_analyzer.py
+++ b/buildscripts/libdeps/graph_analyzer.py
@@ -486,10 +486,9 @@ class UnusedPublicLinter(Analyzer):
 
             if (edge_attribs.get(EdgeProps.direct.name)
                     and edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Public)
-                    and self.graph[edge[0]].get(NodeProps.shim.name)
-                    and self.graph[edge[0]].get(NodeProps.bin_type.name) == 'SharedLibrary'):
-
-                # First we will get all the transitive libdeps the dependnet node
+                    and not self.graph.nodes()[edge[0]].get(NodeProps.shim.name) and
+                    self.graph.nodes()[edge[1]].get(NodeProps.bin_type.name) == 'SharedLibrary'):
+                # First we will get all the transitive libdeps the dependent node
                 # induces, while we are getting those we also check if the depender
                 # node has any symbol dependencies to that transitive libdep.
                 trans_pub_nodes = set([edge[0]])
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 7896d83c0d1..17e2768860a 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -3428,6 +3428,62 @@ tasks:
         targets:
           install-core
 
+- name: libdeps_graph_linting
+  commands:
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        script: |
+          ${activate_virtualenv}
+          python -m pip install networkx
+
+    - func: "scons compile"
+      vars:
+        task_install_action:
+            default
+        task_compile_flags: >-
+          --link-model=dynamic
+          --build-tools=next
+        targets:
+          generate-libdeps-graph
+
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        script: |
+          set -o errexit
+          set -o verbose
+
+          ${activate_virtualenv}
+          GRAPH_FILE=$(find build -name "libdeps.graphml")
+          python buildscripts/libdeps/gacli.py --graph-file $GRAPH_FILE | gzip > results.txt.gz
+          gzip $GRAPH_FILE
+          mv $GRAPH_FILE.gz .
+
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/results.txt.gz
+        remote_file: ${project}/${build_variant}/${revision}/sources/libdeps-${build_id}-results.txt.gz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/gzip}
+        display_name: libdeps results tarball
+
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/libdeps.graphml.gz
+        remote_file: ${project}/${build_variant}/${revision}/sources/libdeps-${build_id}-graph.graphml.gz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/gzip}
+        display_name: libdeps graph tarball
+
 ## compile_all - build all scons targets ##
 - name: compile_all
   depends_on:
@@ -8265,6 +8321,11 @@ task_groups:
   - compile_build_tools_next
 
 - <<: *compile_task_group_template
+  name: libdeps_graph_linting_TG
+  tasks:
+  - libdeps_graph_linting
+
+- <<: *compile_task_group_template
   name: compile_ninja_TG
   tasks:
   - compile_ninja
@@ -8685,6 +8746,9 @@ buildvariants:
   - name: compile_ninja_next_TG
     distros:
     - ubuntu1804-build
+  - name: libdeps_graph_linting_TG
+    distros:
+      - ubuntu1804-build
   - name: compile_ninja_TG
   - name: .aggfuzzer .common
   - name: audit