Merge pull request #1203 from FlorianLudwig/retry_on_network_error

tests: retry on network error (CI)

3 files changed, 77 insertions, 19 deletions

diff --git a/test/helper.py b/test/helper.py
new file mode 100644
index 00000000..240d90c7
--- /dev/null
+++ b/test/helper.py
@@ -0,0 +1,51 @@
+import time
+import urllib.error
+
+import rdflib
+import rdflib.query
+
+
+MAX_RETRY = 10
+BACKOFF_FACTOR = 1.5
+def query_with_retry(graph: rdflib.Graph, query: str, **kwargs) -> rdflib.query.Result:
+    """Query graph an retry on failure, returns preloaded result
+
+    The tests run against outside network targets which results
+    in flaky tests. Therefor retries are needed to increase stability.
+
+    There are two main categories why these might fail:
+
+     * Resource shortage on the server running the tests (e.g. out of ports)
+     * Issues outside the server (network, target server, etc)
+
+    As fast feedback is important the retry should be done quickly.
+    Therefor the first retry is done after 100ms. But if the issue is
+    outside the server running the tests it we need to be good
+    citizenship of the internet and not hit servers of others at 
+    a constant rate. (Also it might get us banned)
+
+    Therefor this function implements a backoff mechanism.
+
+    When adjusting the parameters please keep in mind that several
+    tests might run on the same machine at the same time
+    on our CI, and we really don't want to hit any rate limiting.
+
+    The maximum time the function waits is:
+
+    >>> sum((BACKOFF_FACTOR ** backoff) / 10 for backoff in range(MAX_RETRY))
+    11.3330078125
+    """
+    backoff = 0
+    for i in range(MAX_RETRY):
+        try:
+            result = graph.query(query, **kwargs)
+            result.bindings # access bindings to ensure no lazy loading
+            return result
+        except urllib.error.URLError as e:
+            if i == MAX_RETRY -1:
+              raise e
+
+            backoff_s = (BACKOFF_FACTOR ** backoff) / 10
+            print(f"Network error {e} during query, waiting for {backoff_s:.2f}s and retrying")
+            time.sleep(backoff_s)
+            backoff += 1
diff --git a/test/test_sparql_service.py b/test/test_sparql_service.py
index 550bfcb2..ea1af413 100644
--- a/test/test_sparql_service.py
+++ b/test/test_sparql_service.py
@@ -2,6 +2,8 @@ from rdflib import Graph, URIRef, Literal, Variable
 from rdflib.plugins.sparql import prepareQuery
 from rdflib.compare import isomorphic
 
+from . import helper
+
 
 def test_service():
     g = Graph()
@@ -16,7 +18,7 @@ def test_service():
         <http://www.w3.org/2000/01/rdf-schema#comment> ?dbpComment .
 
     } }  } limit 2"""
-    results = g.query(q)
+    results = helper.query_with_retry(g, q)
     assert len(results) == 2
 
     for r in results:
@@ -38,7 +40,7 @@ def test_service_with_bind():
         <http://dbpedia.org/ontology/deathPlace> ?dbpDeathPlace .
 
     } }  } limit 2"""
-    results = g.query(q)
+    results = helper.query_with_retry(g, q)
     assert len(results) == 2
 
     for r in results:
@@ -60,7 +62,7 @@ def test_service_with_values():
         <http://dbpedia.org/ontology/deathPlace> ?dbpDeathPlace .
 
     } }  } limit 2"""
-    results = g.query(q)
+    results = helper.query_with_retry(g, q)
     assert len(results) == 2
 
     for r in results:
@@ -76,7 +78,7 @@ def test_service_with_implicit_select():
     {
       values (?s ?p ?o) {(<http://example.org/a> <http://example.org/b> 1) (<http://example.org/a> <http://example.org/b> 2)}
     }} limit 2"""
-    results = g.query(q)
+    results = helper.query_with_retry(g, q)
     assert len(results) == 2
 
     for r in results:
@@ -93,7 +95,7 @@ def test_service_with_implicit_select_and_prefix():
     {
       values (?s ?p ?o) {(ex:a ex:b 1) (<http://example.org/a> <http://example.org/b> 2)}
     }} limit 2"""
-    results = g.query(q)
+    results = helper.query_with_retry(g, q)
     assert len(results) == 2
 
     for r in results:
@@ -110,7 +112,7 @@ def test_service_with_implicit_select_and_base():
     {
       values (?s ?p ?o) {(<a> <b> 1) (<a> <b> 2)}
     }} limit 2"""
-    results = g.query(q)
+    results = helper.query_with_retry(g, q)
     assert len(results) == 2
 
     for r in results:
@@ -127,7 +129,7 @@ def test_service_with_implicit_select_and_allcaps():
         ?s <http://purl.org/linguistics/gold/hypernym> <http://dbpedia.org/resource/Leveller> .
       }
     } LIMIT 3"""
-    results = g.query(q)
+    results = helper.query_with_retry(g, q)
     assert len(results) == 3
 
 
diff --git a/test/test_sparqlstore.py b/test/test_sparqlstore.py
index 63b475cc..8154b10c 100644
--- a/test/test_sparqlstore.py
+++ b/test/test_sparqlstore.py
@@ -7,6 +7,9 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 import socket
 from threading import Thread
 
+from . import helper
+
+
 try:
     assert len(urlopen("http://dbpedia.org/sparql").read()) > 0
 except:
@@ -30,7 +33,7 @@ class SPARQLStoreDBPediaTestCase(unittest.TestCase):
 
     def test_Query(self):
         query = "select distinct ?Concept where {[] a ?Concept} LIMIT 1"
-        res = self.graph.query(query, initNs={})
+        res = helper.query_with_retry(self.graph, query, initNs={})
         for i in res:
             assert type(i[0]) == URIRef, i[0].n3()
 
@@ -39,7 +42,7 @@ class SPARQLStoreDBPediaTestCase(unittest.TestCase):
         SELECT ?label WHERE
             { ?s a xyzzy:Concept ; xyzzy:prefLabel ?label . } LIMIT 10
         """
-        res = self.graph.query(
+        res = helper.query_with_retry(self.graph,
             query, initNs={"xyzzy": "http://www.w3.org/2004/02/skos/core#"}
         )
         for i in res:
@@ -60,12 +63,12 @@ class SPARQLStoreDBPediaTestCase(unittest.TestCase):
         SELECT ?label WHERE
             { ?s a xyzzy:Concept ; xyzzy:prefLabel ?label . } LIMIT 10
         """
-        res = self.graph.query(prologue + query)
+        res = helper.query_with_retry(self.graph, prologue + query)
         for i in res:
             assert type(i[0]) == Literal, i[0].n3()
 
     def test_counting_graph_and_store_queries(self):
-        q = """
+        query = """
             SELECT ?s
             WHERE {
                 ?s ?p ?o .
@@ -74,19 +77,21 @@ class SPARQLStoreDBPediaTestCase(unittest.TestCase):
             """
         g = Graph("SPARQLStore")
         g.open(self.path)
-        c = 0
-        for r in g.query(q):
-            c += 1
+        count = 0
+        result = helper.query_with_retry(g, query)
+        for _ in result:
+            count += 1
 
-        assert c == 5, "Graph(\"SPARQLStore\") didn't return 5 records"
+        assert count == 5, "Graph(\"SPARQLStore\") didn't return 5 records"
 
         from rdflib.plugins.stores.sparqlstore import SPARQLStore
         st = SPARQLStore(query_endpoint=self.path)
-        c = 0
-        for r in st.query(q):
-            c += 1
+        count = 0
+        result = helper.query_with_retry(st, query)
+        for _ in result:
+            count += 1
 
-        assert c == 5, "SPARQLStore() didn't return 5 records"
+        assert count == 5, "SPARQLStore() didn't return 5 records"
 
 
 class SPARQLStoreUpdateTestCase(unittest.TestCase):