debuginfod: PR27917 - protect against federation loops

If someone misconfigures a debuginfod federation to have loops, and
a nonexistent buildid lookup is attempted, bad things will happen,
as is documented.

This patch aims to reduce the risk by adding an option to debuginfod
that functions kind of like an IP packet's TTL: a limit on the length of
XFF: header that debuginfod is willing to process. If X-Forwarded-For:
exceeds N hops, it will not delegate a local lookup miss to upstream
debuginfods.

Commit ab38d167c40c99 causes federation loops for non-existent resources
to result in multiple temporary deadlocks, each lasting for
$DEBUGINFOD_TIMEOUT seconds. Since concurrent requests for each unique
resource are now serialized, federation loops can result in one server
thread waiting to acquire a lock while the server thread holding the
lock waits for the first thread to respond to an http request.

This PR can help protect against the above multiple temporary deadlocks
behaviour. Ex. if --forwarded-ttl-limit=0 then the timeout behaviour of
local loops should be avoided.

https://sourceware.org/bugzilla/show_bug.cgi?id=27917

Signed-off-by: Di Chen <dichen@redhat.com>

6 files changed, 86 insertions, 1 deletions

diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index e8e486ab..395af94f 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,11 @@
+2021-08-20  Di Chen  <dichen@redhat.com>
+
+	* debuginfod.cxx (options): Add ARGP_KEY_FORWARDED_TTL_LIMIT.
+	(forwarded_ttl_limit): New static unsigned.
+	(parse_opt): Handle ARGP_KEY_FORWARDED_TTL_LIMIT.
+	(handle_buildid): Check forwarded_ttl_limit.
+	(main): Log forwarded ttl limit.
+
 2021-08-20  Saleem Abdulrasool  <abdulras@google.com>
 
 	* debuginfod.cxx: Remove error.h include.
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index b560fdcb..6e182a84 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -374,6 +374,8 @@ static const struct argp_option options[] =
 #define ARGP_KEY_FDCACHE_PREFETCH_FDS 0x1006
    { "fdcache-prefetch-fds", ARGP_KEY_FDCACHE_PREFETCH_FDS, "NUM", 0,"Number of files allocated to the \
       prefetch cache.", 0},
+#define ARGP_KEY_FORWARDED_TTL_LIMIT 0x1007
+   {"forwarded-ttl-limit", ARGP_KEY_FORWARDED_TTL_LIMIT, "NUM", 0, "Limit of X-Forwarded-For hops, default 8.", 0},
    { NULL, 0, NULL, 0, NULL, 0 },
   };
 
@@ -421,6 +423,7 @@ static long fdcache_prefetch;
 static long fdcache_mintmp;
 static long fdcache_prefetch_mbs;
 static long fdcache_prefetch_fds;
+static unsigned forwarded_ttl_limit = 8;
 static string tmpdir;
 
 static void set_metric(const string& key, double value);
@@ -553,6 +556,9 @@ parse_opt (int key, char *arg,
       if( fdcache_mintmp > 100 || fdcache_mintmp < 0 )
         argp_failure(state, 1, EINVAL, "fdcache mintmp percent");
       break;
+    case ARGP_KEY_FORWARDED_TTL_LIMIT:
+      forwarded_ttl_limit = (unsigned) atoi(arg);
+      break;
     case ARGP_KEY_ARG:
       source_paths.insert(string(arg));
       break;
@@ -1880,6 +1886,17 @@ handle_buildid (MHD_Connection* conn,
           if (xff != "")
             xff += string(", "); // comma separated list
 
+          unsigned int xff_count = 0;
+          for (auto&& i : xff){
+            if (i == ',') xff_count++;
+          }
+
+          // if X-Forwarded-For: exceeds N hops,
+          // do not delegate a local lookup miss to upstream debuginfods.
+          if (xff_count >= forwarded_ttl_limit)
+            throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found, --forwared-ttl-limit reached \
+and will not query the upstream servers");
+
           // Compute the client's numeric IP address only - so can't merge with conninfo()
           const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
                                                                        MHD_CONNECTION_INFO_CLIENT_ADDRESS);
@@ -3718,6 +3735,7 @@ main (int argc, char *argv[])
   obatched(clog) << "groom time " << groom_s << endl;
   obatched(clog) << "prefetch fds " << fdcache_prefetch_fds << endl;
   obatched(clog) << "prefetch mbs " << fdcache_prefetch_mbs << endl;
+  obatched(clog) << "forwarded ttl limit " << forwarded_ttl_limit << endl;
 
   if (scan_archives.size()>0)
     {
diff --git a/doc/ChangeLog b/doc/ChangeLog
index 77d1d705..d5f34f0f 100644
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+2021-08-20  Di Chen  <dichen@redhat.com>
+
+	* debuginfod.8: Add --forwarded-ttl-limit=NUM documentation.
+
 2021-07-28  Alice Zhang <alizhang@redhat.com>
 
 	PR27950
diff --git a/doc/debuginfod.8 b/doc/debuginfod.8
index 3e791b62..5b0d793c 100644
--- a/doc/debuginfod.8
+++ b/doc/debuginfod.8
@@ -236,6 +236,12 @@ can translate to RAM scarcity if the disk happens to be on a RAM
 virtual disk.  The default threshold is 25%.
 
 .TP
+.B "\-\-forwarded\-ttl\-limit=NUM"
+Configure limits of X-Forwarded-For hops. if X-Forwarded-For
+exceeds N hops, it will not delegate a local lookup miss to
+upstream debuginfods. The default limit is 8.
+
+.TP
 .B "\-v"
 Increase verbosity of logging to the standard error file descriptor.
 May be repeated to increase details.  The default verbosity is 0.
diff --git a/tests/ChangeLog b/tests/ChangeLog
index 3bfd1ca2..29c48b97 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,7 @@
+2021-08-20  Di Chen  <dichen@redhat.com>
+
+	* run-debuginfod-find.sh: Add test for X-Forwarded-For hops limit.
+
 2021-08-04  Mark Wielaard  <mark@klomp.org>
 
 	PR28190
diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh
index 991d1dc5..7e12dd7f 100755
--- a/tests/run-debuginfod-find.sh
+++ b/tests/run-debuginfod-find.sh
@@ -37,6 +37,8 @@ PID1=0
 PID2=0
 PID3=0
 PID4=0
+PID5=0
+PID6=0
 
 cleanup()
 {
@@ -44,6 +46,8 @@ cleanup()
   if [ $PID2 -ne 0 ]; then kill $PID2; wait $PID2; fi
   if [ $PID3 -ne 0 ]; then kill $PID3; wait $PID3; fi
   if [ $PID4 -ne 0 ]; then kill $PID4; wait $PID4; fi
+  if [ $PID5 -ne 0 ]; then kill $PID5; wait $PID5; fi
+  if [ $PID6 -ne 0 ]; then kill $PID6; wait $PID6; fi
   rm -rf F R D L Z ${PWD}/foobar ${PWD}/mocktree ${PWD}/.client_cache* ${PWD}/tmp*
   exit_cleanup
 }
@@ -54,7 +58,7 @@ trap cleanup 0 1 2 3 5 9 15
 errfiles_list=
 err() {
     echo ERROR REPORTS
-    for ports in $PORT1 $PORT2 $PORT3
+    for ports in $PORT1 $PORT2 $PORT3 $PORT4 $PORT5
     do
         echo ERROR REPORT $port metrics
         curl -s http://127.0.0.1:$port/metrics
@@ -804,4 +808,45 @@ if [ $retry_attempts -ne 10 ]; then
     exit 1;
   fi
 
+########################################################################
+
+# Test when debuginfod hitting X-Forwarded-For hops limit.
+# This test will start two servers (as a loop) with two different hop limits.
+
+while true; do
+    PORT4=`expr '(' $RANDOM % 1000 ')' + 9000`
+    PORT5=`expr '(' $RANDOM % 1000 ')' + 9000`
+    ss -atn | fgrep -e ":$PORT4" -e ":$PORT5"|| break
+done
+
+# Make sure the vlogs are cleaned up after the test
+# and that they are printed on error.
+tempfiles vlog$PORT4 vlog$PORT5
+errfiles vlog$PORT4 vlog$PORT5
+
+env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS=http://127.0.0.1:$PORT5 ${abs_builddir}/../debuginfod/debuginfod $VERBOSE --forwarded-ttl-limit 0 -p $PORT4 > vlog$PORT4 2>&1 &
+PID5=$!
+
+env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS=http://127.0.0.1:$PORT4 ${abs_builddir}/../debuginfod/debuginfod $VERBOSE --forwarded-ttl-limit 1 -p $PORT5 > vlog$PORT5 2>&1 &
+PID6=$!
+
+wait_ready $PORT4 'ready' 1
+wait_ready $PORT5 'ready' 1
+
+export DEBUGINFOD_URLS="http://127.0.0.1:$PORT4/"
+testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo 01234567 || true
+
+# Use a different buildid to avoid using same cache.
+export DEBUGINFOD_URLS="http://127.0.0.1:$PORT5/"
+testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo 11234567 || true
+
+grep "forwared-ttl-limit reached and will not query the upstream servers" vlog$PORT4
+grep -v "forwared-ttl-limit reached and will not query the upstream servers" vlog$PORT5 | grep "not found" vlog$PORT5
+
+kill $PID5 $PID6
+wait $PID5 $PID6
+
+PID5=0
+PID6=0
+
 exit 0