Improve performance and memory usage of project export

ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.

To improve this, we can do several things:

1. We use `tree:` and `preload:` to automatically generate
   a list of all preloads that could be used to serialize
   objects in bulk.

2. We observe that a single project has many issues, merge requests,
   etc. Instead of serializing everything at once, which could lead to
   database timeouts and high memory usage, we take each top-level
   association and serialize the data in batches.

For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.

1 files changed, 51 insertions, 16 deletions

diff --git a/spec/lib/gitlab/import_export/attributes_finder_spec.rb b/spec/lib/gitlab/import_export/attributes_finder_spec.rb
index 208b60844e3..3cbc1375d6e 100644
--- a/spec/lib/gitlab/import_export/attributes_finder_spec.rb
+++ b/spec/lib/gitlab/import_export/attributes_finder_spec.rb
@@ -20,20 +20,41 @@ describe Gitlab::ImportExport::AttributesFinder do
             except: [:iid],
             include: [
               { merge_request_diff: {
-                include: []
+                include: [],
+                preload: { source_project: nil }
               } },
               { merge_request_test: { include: [] } }
             ],
-            only: [:id]
+            only: [:id],
+            preload: {
+              merge_request_diff: { source_project: nil },
+              merge_request_test: nil
+            }
             } },
           { commit_statuses: {
-              include: [{ commit: { include: [] } }]
+              include: [{ commit: { include: [] } }],
+              preload: { commit: nil }
             } },
           { project_members: {
               include: [{ user: { include: [],
-                                  only: [:email] } }]
+                                  only: [:email] } }],
+              preload: { user: nil }
           } }
-        ]
+        ],
+        preload: {
+          commit_statuses: {
+            commit: nil
+          },
+          issues: nil,
+          labels: nil,
+          merge_requests: {
+            merge_request_diff: { source_project: nil },
+            merge_request_test: nil
+          },
+          project_members: {
+            user: nil
+          }
+        }
       }
     end
 
@@ -50,7 +71,8 @@ describe Gitlab::ImportExport::AttributesFinder do
         setup_yaml(tree: { project: [:issues] })
 
         is_expected.to match(
-          include: [{ issues: { include: [] } }]
+          include: [{ issues: { include: [] } }],
+          preload: { issues: nil }
         )
       end
 
@@ -58,7 +80,8 @@ describe Gitlab::ImportExport::AttributesFinder do
         setup_yaml(tree: { project: [:project_feature] })
 
         is_expected.to match(
-          include: [{ project_feature: { include: [] } }]
+          include: [{ project_feature: { include: [] } }],
+          preload: { project_feature: nil }
         )
       end
 
@@ -67,7 +90,8 @@ describe Gitlab::ImportExport::AttributesFinder do
 
         is_expected.to match(
           include: [{ issues: { include: [] } },
-                    { snippets: { include: [] } }]
+                    { snippets: { include: [] } }],
+          preload: { issues: nil, snippets: nil }
         )
       end
 
@@ -75,7 +99,9 @@ describe Gitlab::ImportExport::AttributesFinder do
         setup_yaml(tree: { project: [issues: [:notes]] })
 
         is_expected.to match(
-          include: [{ issues: { include: [{ notes: { include: [] } }] } }]
+          include: [{ issues: { include: [{ notes: { include: [] } }],
+                                preload: { notes: nil } } }],
+          preload: { issues: { notes: nil } }
         )
       end
 
@@ -85,7 +111,9 @@ describe Gitlab::ImportExport::AttributesFinder do
         is_expected.to match(
           include: [{ merge_requests:
                       { include: [{ notes: { include: [] } },
-                                  { merge_request_diff: { include: [] } }] } }]
+                                  { merge_request_diff: { include: [] } }],
+                        preload: { merge_request_diff: nil, notes: nil } } }],
+          preload: { merge_requests: { merge_request_diff: nil, notes: nil } }
         )
       end
 
@@ -94,8 +122,11 @@ describe Gitlab::ImportExport::AttributesFinder do
 
         is_expected.to match(
           include: [{ merge_requests: {
-                      include: [{ notes: { include: [{ author: { include: [] } }] } }]
-                    } }]
+                      include: [{ notes: { include: [{ author: { include: [] } }],
+                                          preload: { author: nil } } }],
+                      preload: { notes: { author: nil } }
+                    } }],
+          preload: { merge_requests: { notes: { author: nil } } }
         )
       end
 
@@ -105,7 +136,8 @@ describe Gitlab::ImportExport::AttributesFinder do
 
         is_expected.to match(
           include: [{ issues: { include: [],
-                                only: [:name, :description] } }]
+                                only: [:name, :description] } }],
+          preload: { issues: nil }
         )
       end
 
@@ -115,7 +147,8 @@ describe Gitlab::ImportExport::AttributesFinder do
 
         is_expected.to match(
           include: [{ issues: { except: [:name],
-                                include: [] } }]
+                                include: [] } }],
+          preload: { issues: nil }
         )
       end
 
@@ -127,7 +160,8 @@ describe Gitlab::ImportExport::AttributesFinder do
         is_expected.to match(
           include: [{ issues: { except: [:name],
                                 include: [],
-                                only: [:description] } }]
+                                only: [:description] } }],
+          preload: { issues: nil }
         )
       end
 
@@ -137,7 +171,8 @@ describe Gitlab::ImportExport::AttributesFinder do
 
         is_expected.to match(
           include: [{ issues: { include: [],
-                                methods: [:name] } }]
+                                methods: [:name] } }],
+          preload: { issues: nil }
         )
       end