Merge branch 'juerg/reapi-platform' into 'master'

_sandboxreapi.py: Set platform properties

See merge request BuildStream/buildstream!1823

33 files changed, 72 insertions, 41 deletions

diff --git a/.gitlab-ci/buildgrid-compose.yml b/.gitlab-ci/buildgrid-compose.yml
index 2d2725706..fd45c45af 100644
--- a/.gitlab-ci/buildgrid-compose.yml
+++ b/.gitlab-ci/buildgrid-compose.yml
@@ -38,7 +38,7 @@ services:
   bot:
     image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildbox:nightly
     command: [
-      "sh", "-c", "sleep 15 && ( buildbox-casd --cas-remote=http://controller:50051 /var/lib/buildgrid/cache & buildbox-worker --bots-remote=http://controller:50051 --cas-remote=unix:/var/lib/buildgrid/cache/casd.sock --buildbox-run=buildbox-run-bubblewrap --runner-arg=--use-localcas --verbose )"]
+      "sh", "-c", "sleep 15 && ( buildbox-casd --cas-remote=http://controller:50051 /var/lib/buildgrid/cache & buildbox-worker --bots-remote=http://controller:50051 --cas-remote=unix:/var/lib/buildgrid/cache/casd.sock --buildbox-run=buildbox-run-bubblewrap --runner-arg=--use-localcas --platform OSFamily=linux --platform ISA=x86-64 --verbose )"]
     privileged: true
     volumes:
       - type: volume
diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py
index c838ef6c2..e0a0cf7ce 100644
--- a/src/buildstream/_platform/platform.py
+++ b/src/buildstream/_platform/platform.py
@@ -138,7 +138,13 @@ class Platform:
 
     @staticmethod
     def get_host_os():
-        return platform.uname().system
+        system = platform.uname().system.lower()
+
+        if system == "darwin" and platform.mac_ver()[0]:
+            # mac_ver() returns a non-empty release string on macOS.
+            return "macos"
+        else:
+            return system
 
     # canonicalize_arch():
     #
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 071d085b8..f270bd8cc 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -2721,6 +2721,12 @@ class Element(Plugin):
         # Sandbox config, unlike others, has fixed members so we should validate them
         sandbox_config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch"])
 
+        build_os = sandbox_config.get_str("build-os", default=None)
+        if build_os:
+            build_os = build_os.lower()
+        else:
+            build_os = host_os
+
         build_arch = sandbox_config.get_str("build-arch", default=None)
         if build_arch:
             build_arch = Platform.canonicalize_arch(build_arch)
@@ -2728,10 +2734,7 @@ class Element(Plugin):
             build_arch = host_arch
 
         return SandboxConfig(
-            sandbox_config.get_int("build-uid"),
-            sandbox_config.get_int("build-gid"),
-            sandbox_config.get_str("build-os", default=host_os),
-            build_arch,
+            sandbox_config.get_int("build-uid"), sandbox_config.get_int("build-gid"), build_os, build_arch,
         )
 
     # This makes a special exception for the split rules, which
diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py b/src/buildstream/sandbox/_sandboxbuildboxrun.py
index 93ee330b1..df0703dfd 100644
--- a/src/buildstream/sandbox/_sandboxbuildboxrun.py
+++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py
@@ -56,10 +56,6 @@ class SandboxBuildBoxRun(SandboxREAPI):
 
     @classmethod
     def check_sandbox_config(cls, platform, config):
-        # Report error for elements requiring non-0 UID/GID
-        if config.build_uid != 0 or config.build_gid != 0:
-            return False
-
         # Check host os and architecture match
         if config.build_os != platform.get_host_os():
             raise SandboxError("Configured and host OS don't match.")
@@ -176,5 +172,8 @@ class SandboxBuildBoxRun(SandboxREAPI):
             if returncode != 0:
                 raise SandboxError("buildbox-run failed with returncode {}".format(returncode))
 
+    def _supported_platform_properties(self):
+        return {"unixUID", "unixGID", "network"}
+
     def _warn(self, msg):
         self._get_context().messenger.message(Message(MessageType.WARN, msg))
diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py
index 2f500cd51..19c599197 100644
--- a/src/buildstream/sandbox/_sandboxreapi.py
+++ b/src/buildstream/sandbox/_sandboxreapi.py
@@ -78,7 +78,7 @@ class SandboxREAPI(Sandbox):
 
         # Generate Action proto
         input_root_digest = vdir._get_digest()
-        command_proto = self._create_command(command, cwd, env, read_write_directories)
+        command_proto = self._create_command(command, cwd, env, read_write_directories, flags)
         command_digest = cascache.add_object(buffer=command_proto.SerializeToString())
         action = remote_execution_pb2.Action(command_digest=command_digest, input_root_digest=input_root_digest)
         if self._output_node_properties:
@@ -95,7 +95,7 @@ class SandboxREAPI(Sandbox):
         # the remote execution system has worked correctly but the command failed.
         return action_result.exit_code
 
-    def _create_command(self, command, working_directory, environment, read_write_directories):
+    def _create_command(self, command, working_directory, environment, read_write_directories, flags):
         # Creates a command proto
         environment_variables = [
             remote_execution_pb2.Command.EnvironmentVariable(name=k, value=v) for (k, v) in environment.items()
@@ -104,13 +104,33 @@ class SandboxREAPI(Sandbox):
         # Request read-write directories as output
         output_directories = [os.path.relpath(dir, start=working_directory) for dir in read_write_directories]
 
+        config = self._get_config()
+        supported_properties = self._supported_platform_properties()
+        platform = remote_execution_pb2.Platform()
+        platform.properties.add(name="OSFamily", value=config.build_os)
+        platform.properties.add(name="ISA", value=config.build_arch)
+
+        if flags & SandboxFlags.INHERIT_UID:
+            uid = os.geteuid()
+            gid = os.getegid()
+        else:
+            uid = config.build_uid
+            gid = config.build_gid
+        if "unixUID" in supported_properties:
+            platform.properties.add(name="unixUID", value=str(uid))
+        if "unixGID" in supported_properties:
+            platform.properties.add(name="unixGID", value=str(gid))
+
+        if flags & SandboxFlags.NETWORK_ENABLED and "network" in supported_properties:
+            platform.properties.add(name="network", value="on")
+
         return remote_execution_pb2.Command(
             arguments=command,
             working_directory=working_directory[1:],
             environment_variables=environment_variables,
             output_files=[],
             output_directories=output_directories,
-            platform=None,
+            platform=platform,
         )
 
     def _process_job_output(self, working_directory, output_directories, output_files, *, failure):
@@ -153,6 +173,9 @@ class SandboxREAPI(Sandbox):
     def _execute_action(self, action, flags):
         raise ImplError("Sandbox of type '{}' does not implement _execute_action()".format(type(self).__name__))
 
+    def _supported_platform_properties(self):
+        return set()
+
 
 # _SandboxREAPIBatch()
 #
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index e3653d845..caa45bf73 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-87391ccead1223d376cbc5261a42a163762f895d5c035430084b9e419b004174
\ No newline at end of file
+648848a7fba41a9da8dec2e0aeb00fc011aec529c791c8c29d9a47fab795e8ef
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index e5f7e165c..40520a7aa 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-55eff1fd81182b3a441d5e587d99ceff025d866bbe614260b3aa8b8356eb79aa
\ No newline at end of file
+af612783a2d9c3b4f1fc00b79094eb707c11b311d33acf9c737f1021b6c4ff9b
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index 3ceee5957..f914caba8 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-a5bcbca1752d8378cbd1a85ea93f4ecf7a74476fa9cd81f899c2cdc2a5f4fb43
\ No newline at end of file
+80f5d8706bb6e1eb794e74a78d0738850dae3155091879525860c979c3cdf696
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index c906f88f5..307d33076 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-07f80eaf2c6de2e019b64ecccae1dd45696427d6a6d05680e429785fa341e318
\ No newline at end of file
+e9c86ffe988cce20c1226a40bed0af44400d3ddc1a5942424aa43202781069b0
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index b795a5bd3..d02882afd 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-1fed855571867d0b1bd7f84128ad21aab60550a7bd6b32c679b093c1d3f8fdb8
\ No newline at end of file
+b79b0265dadbd9a052c677f85615bec95153055eb64091378fff8eb3dc9ecae4
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index c9986deed..750423f70 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-8f1a61a35a40e968acfaf298403f8546f80d63597954a974a261d8aea1a715e7
\ No newline at end of file
+3cd0532d8160db2918970133dc7d7e4ddf0fd4f5aa03340753d941ab894e2c8f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 214600317..8aa52e9f5 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-96622c2c119879119c54412437eba4d8b4e906c5d27bae1e7bffc1da3a2a6b5e
\ No newline at end of file
+b545838260f214d6a774cb0757e5fdbf81041924cf0036e335dee6a5a2fb96c4
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index 78f4972fa..e242882a3 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file
+01814090d495860a7ab39054f57b32197661e4d5ba1b9573c12cc0edeecb75a8
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 52b074d6c..802e1347a 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-1895fc7749e6839c26b881f1957e8778e7a459ef1ffaca8cf409d45324330362
\ No newline at end of file
+fc2ce4996757fa4557454a10c1f73e746604fae096c226adf05642559f35763e
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 9a0a4ae66..438bf9172 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-a51f965f654e70ef4630d2f6cbaeb8c9f27bc190d3c7827e0e95feb4fc83b944
\ No newline at end of file
+a828a8bfb3b036441dcbf635ca4ef6e7f079cb52128cacf4a8d844cb44e3c762
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 51e5b97e1..6667b5bea 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-00788f98d62eb5fafac1fc1bd52530fcb8f03091edf205b308c61d2b621bebe3
\ No newline at end of file
+472cfe8a41856db06bf1d5c2fa052e8985ce00534df4a98b22378f15b2d9d188
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 253ebc601..50eff30c6 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-673bb938cc3fabe0be55e98c6b8b80853168becc86b4fa102fc0c538879bf83a
+2825e2d55baa6617ec7e559ae554bb66761105a1ce50028bffdb983b6bb11707
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index c8e11137a..833e3a9a9 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-53a367133fb8f3ca86ba772801ea62681414271da9582800dd56a62e9c6d7e5d
+4e21bc1089066998d44933fdf1e48b2a5a73ceaba7a3944538f3f142e34b8b8e
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 2e320c5ff..b4d2e70ae 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-86e0cf4f3154fa006899acf64317930ebf08ca6d01113abfa35ccceed2961fcd
+c6cb011849cb0ded98add6f46e6737caa1191e6cec58bcee20f06a5198f03466
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index fd1b1d8d0..2d12c164d 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-ad5dff8a422c9de7c3d02773aeed7b425d43501ee5c2d5d13064b6f1e1ed9dec
+65f9f934ed84d63bb19dab4c323bb3a4a3cef85d9592f5be9d6d6e420952747e
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 78f4972fa..e242882a3 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file
+01814090d495860a7ab39054f57b32197661e4d5ba1b9573c12cc0edeecb75a8
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index e32afb042..184ededc7 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-427439145ad4a9c3b839f6db8fc81d693d0163da5f3e48f36e6f1e39fc5d816b
\ No newline at end of file
+586f82dd991878c0bd4b373bcd9685505c930cf2477b13557257b962c899ef70
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index f4a5b6318..873cb1f21 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-4d2d8e8e92a20255a38d167abd93b5e6843f7b2738cdfe11ce64bc662fcaa886
+7bc6295e5f201f2a04ccd7ea8b3be8ef904fa684bfb5d6ec3ec37174bb4a4cf3
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 99a1923a2..a9deef92f 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-21a53c232671f21cd717a4c94274e2decdba2c916dde56e030f944fe92ae785e
+973765368a6082d2c317da1f1fffbcf1f1ab586548d816a6e7a1f6136f3271aa
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index b44d06530..b04dc3b75 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-5ec3023b14bb2a44c94e205d4edc0e366d187357d6661bbc699f73e014b0630b
+506ccb32c5d9960fa74a3e224c583a399a71198162fb7c5604cb8d4b53af2307
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index e5ba434ab..844f4c54c 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-60d6200ba331e3cff4b3255cb218569e387c571ee57761f6b3883b1283a937a2
+70870ffa81e4527a2d812c809b1e99025c72d1ae289be38e45a5b22dc0262eac
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 2f42a3356..6f735034a 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-46f09da7ea078bf0d630ec2e14a668f8144df5175ee1c19c9af367873047b482
+94cac120225f3c023bdf3933b1248dbd1a0cdc401af1d0d76a3f603a1928c863
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 1dbaf7739..f575c8db3 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-aaa2d0c22b40d2f9b87d40ff24c37769240edba4902c50fa948e8ab6c9848f6f
+51e296dd94a8e8fd94c85074458597b6ff33679add366a213187bd3904c1018a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 153138437..6dda711fd 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-6b550e20ab7b8a11912ca14171e39c76badf7fa161a01c83d817c789b84e45c3
+a1cb6f9f795c29d8c73239eff44bdabac29ee31c64cb1b6afb599093b9ce8737
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 6440dfaed..6fb5021d5 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-f890b611cc83036b9c52dddf4eb2a02ccac5a73ae3ddcb34586406d7deba5a11
+86f177f46e35ff192796d4523d1d9537d518dde93f8f2ceba132750f0661706a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 1207db1a7..e34a14eaf 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-5393d513abcc88bd1cdbf03cff65d470285a906a43cf2e192ce0770fbceb933d
+a409fc675bb26b64b1670abbb9faf8baa64ed04ea291fa7424ee549d915f90f0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 4fa2e563d..dce49654a 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-a03196c4878e0a585c54c0e75cabe069068d5e37b49f07ca95f5aeb6e3b1cf5b
+1f59dfd67201062654ddf17394d026b85de0f7bdaf949f857e324ea202f34b1a
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 17d7376dc..cf38ddcfe 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-9fba84d9e7fbbb03121e4bc24fb58dd9a101c88abdb0437c82402e342d46b88f
\ No newline at end of file
+a57f84353d1528b3ecbd9993ac02a20d0355f2451e19700172c95575af9a5b48
\ No newline at end of file