From c2c10d68a5e3ed12d13adce2cb10feed789f8f8c Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Thu, 14 Feb 2019 11:25:10 +0000 Subject: Cache Keys: Update to use JSON rather than pickle This affects the cache key version (updated to 7) and introduces a dependency on `ujson` which is BSD licenced as of the version locked in `requirements.txt` Signed-off-by: Daniel Silverstone --- buildstream/_cachekey.py | 7 ++++--- buildstream/_versions.py | 2 +- requirements/requirements.in | 1 + requirements/requirements.txt | 1 + tests/cachekey/project/elements/build1.expected | 2 +- tests/cachekey/project/elements/build2.expected | 2 +- tests/cachekey/project/elements/compose1.expected | 2 +- tests/cachekey/project/elements/compose2.expected | 2 +- tests/cachekey/project/elements/compose3.expected | 2 +- tests/cachekey/project/elements/compose4.expected | 2 +- tests/cachekey/project/elements/compose5.expected | 2 +- tests/cachekey/project/elements/import1.expected | 2 +- tests/cachekey/project/elements/import2.expected | 2 +- tests/cachekey/project/elements/import3.expected | 2 +- tests/cachekey/project/elements/script1.expected | 2 +- tests/cachekey/project/sources/bzr1.expected | 2 +- tests/cachekey/project/sources/git1.expected | 2 +- tests/cachekey/project/sources/git2.expected | 2 +- tests/cachekey/project/sources/git3.expected | 2 +- tests/cachekey/project/sources/local1.expected | 2 +- tests/cachekey/project/sources/local2.expected | 2 +- tests/cachekey/project/sources/ostree1.expected | 2 +- tests/cachekey/project/sources/patch1.expected | 2 +- tests/cachekey/project/sources/patch2.expected | 2 +- tests/cachekey/project/sources/patch3.expected | 2 +- tests/cachekey/project/sources/pip1.expected | 2 +- tests/cachekey/project/sources/remote1.expected | 2 +- tests/cachekey/project/sources/remote2.expected | 2 +- tests/cachekey/project/sources/tar1.expected | 2 +- tests/cachekey/project/sources/tar2.expected | 2 +- tests/cachekey/project/sources/zip1.expected | 2 +- tests/cachekey/project/sources/zip2.expected | 2 +- tests/cachekey/project/target.expected | 2 +- 33 files changed, 36 insertions(+), 33 deletions(-) diff --git a/buildstream/_cachekey.py b/buildstream/_cachekey.py index fe407e96f..953263aa9 100644 --- a/buildstream/_cachekey.py +++ b/buildstream/_cachekey.py @@ -19,7 +19,8 @@ import hashlib -import pickle + +import ujson from . import _yaml @@ -38,5 +39,5 @@ from . import _yaml # def generate_key(value): ordered = _yaml.node_sanitize(value) - string = pickle.dumps(ordered) - return hashlib.sha256(string).hexdigest() + ustring = ujson.dumps(ordered, sort_keys=True, escape_forward_slashes=False).encode('utf-8') + return hashlib.sha256(ustring).hexdigest() diff --git a/buildstream/_versions.py b/buildstream/_versions.py index 2563fa504..f24a74714 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 21 # or if buildstream was changed in a way which can cause # the same cache key to produce something that is no longer # the same. -BST_CORE_ARTIFACT_VERSION = 6 +BST_CORE_ARTIFACT_VERSION = 7 diff --git a/requirements/requirements.in b/requirements/requirements.in index 45e7b8075..4f08969bb 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -14,3 +14,4 @@ psutil ruamel.yaml >= 0.15.41, < 0.15.52 setuptools pyroaring +ujson diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 115879ad9..d46f92553 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -14,6 +14,7 @@ psutil==5.4.8 ruamel.yaml==0.15.51 setuptools==39.0.1 pyroaring==0.2.6 +ujson==1.35 ## The following requirements were added by pip freeze: MarkupSafe==1.1.0 six==1.12.0 diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected index c2982f939..901f6acf7 100644 --- a/tests/cachekey/project/elements/build1.expected +++ b/tests/cachekey/project/elements/build1.expected @@ -1 +1 @@ -298bb7caab56c60bc12f59df15db2b1107a604fdb0ce3fa5729aa23d6d37f6e4 \ No newline at end of file +0eddcb5c12b2bdb4ce83bef7b997f25c89a9396c6321fe471b31dabba8eba928 \ No newline at end of file diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected index bc663989f..cfff92ad0 100644 --- a/tests/cachekey/project/elements/build2.expected +++ b/tests/cachekey/project/elements/build2.expected @@ -1 +1 @@ -c745f07458a2488180ae7f5b0601a38853dcdfd2b6771c18ecbb622f1af10cf1 \ No newline at end of file +d54339437d5495f573a01434c61b8f0496cef0f833566b9c9f0a6cb362075edb \ No newline at end of file diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected index 9abf5b751..d8f256190 100644 --- a/tests/cachekey/project/elements/compose1.expected +++ b/tests/cachekey/project/elements/compose1.expected @@ -1 +1 @@ -d759c8b7262c4782c12b425bfca0bc8f4c30656d91c98e309c10f11d4179b101 \ No newline at end of file +d14a3d59da4fcb18c81f0cd6b5e893dc5f25e1d5eecc1a0845fb9b2ef499d878 \ No newline at end of file diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected index 4c3881b8e..f7de6d183 100644 --- a/tests/cachekey/project/elements/compose2.expected +++ b/tests/cachekey/project/elements/compose2.expected @@ -1 +1 @@ -01e22d37b82c9b35c35d3d8485e05fe6de526d3af00137bbff51a309b1fa3401 \ No newline at end of file +29857451bb3797e258911e485f94c7d59e13f239f3ac904e7524c4d0a187e240 \ No newline at end of file diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected index e66de59f5..0cf041baf 100644 --- a/tests/cachekey/project/elements/compose3.expected +++ b/tests/cachekey/project/elements/compose3.expected @@ -1 +1 @@ -22b846d1928155b5a21497e8048af2945ee03a826338f91d4f00d74597cc3ef7 \ No newline at end of file +2dbb668ebc50445dc03cd44d6d1110cc14d50093a37ef4a25499055318835174 \ No newline at end of file diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected index abc56f234..db0879543 100644 --- a/tests/cachekey/project/elements/compose4.expected +++ b/tests/cachekey/project/elements/compose4.expected @@ -1 +1 @@ -0e00f203ce9bc4163be0ed304fcf2abb6ee39aad4ee9a12d80719708a4327337 \ No newline at end of file +7bc4f89c2a4c27999626c9c8a28fd2865695588375660236e3aec49e847705c1 \ No newline at end of file diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected index 83345705e..8300b0302 100644 --- a/tests/cachekey/project/elements/compose5.expected +++ b/tests/cachekey/project/elements/compose5.expected @@ -1 +1 @@ -3d8dd9fef508c27961edeb21dd469d04e7ea097c86a98e7c3bf725975548748f \ No newline at end of file +12753c08f60587f22f4ef877b636f6e237f958233ea9b512471de213c84ff927 \ No newline at end of file diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected index fd349363e..32f292fd0 100644 --- a/tests/cachekey/project/elements/import1.expected +++ b/tests/cachekey/project/elements/import1.expected @@ -1 +1 @@ -02f863aedaf175f4b4bb9979a538bd1fb6eabd0ef9d77a633b141c072d957df5 \ No newline at end of file +bbe25b17a5190532ab36663eb08d3f855a45e09108a80ef619a1198e9d730dcf \ No newline at end of file diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected index e68e2b505..f271519e0 100644 --- a/tests/cachekey/project/elements/import2.expected +++ b/tests/cachekey/project/elements/import2.expected @@ -1 +1 @@ -80ee70ad27ee4640400533585d1b2313b1abdb0034f4159dad42f545310d9c24 \ No newline at end of file +532277cf8cf2b5d609bbd690dd3cf2eb2e1042329cdd4b25f7a03f77209bf316 \ No newline at end of file diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected index 1cab93718..075d6af90 100644 --- a/tests/cachekey/project/elements/import3.expected +++ b/tests/cachekey/project/elements/import3.expected @@ -1 +1 @@ -b30d79b763ba3048945784d06482ee378d2642dff712ecffae737aa6f9391b9f \ No newline at end of file +5b5911dad04b5f592cedd29b46aa55b87428c31b2bf6539f9c2b2e51a932adbe \ No newline at end of file diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected index 18977ecc0..58f79dde5 100644 --- a/tests/cachekey/project/elements/script1.expected +++ b/tests/cachekey/project/elements/script1.expected @@ -1 +1 @@ -fb69c9237156bdeb0f948078f735189e9d8a4c4dca3c67b6f1ed5fa314f88707 \ No newline at end of file +818bda0985ef569204c392de0ae2e7ebefd8fcac94f0c33226270936978d52ee \ No newline at end of file diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected index 68ef31975..dd527da83 100644 --- a/tests/cachekey/project/sources/bzr1.expected +++ b/tests/cachekey/project/sources/bzr1.expected @@ -1 +1 @@ -5a8144078d50ff5507d7260c2aa7c62fc6272554767392cd827277e224385cb9 \ No newline at end of file +767d481ad88648c7a926190ac6697c2d24c563c5133abab9e6ab16c6156941d3 \ No newline at end of file diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected index 271faafdf..9a6d7c991 100644 --- a/tests/cachekey/project/sources/git1.expected +++ b/tests/cachekey/project/sources/git1.expected @@ -1 +1 @@ -21968271111bb43e57417d24481742c2fe52fddd798af4aacca0f604c54b6b94 \ No newline at end of file +4ccd8fdfcca973bd9079960f260c22df8bef5df7f598009f0ee896d3b5905b6e \ No newline at end of file diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected index 8abecc0c0..54cc4bff3 100644 --- a/tests/cachekey/project/sources/git2.expected +++ b/tests/cachekey/project/sources/git2.expected @@ -1 +1 @@ -f3d0b8c1179f7b7a22beb4cef7b33a13af047bd882e7e3c34cea55c85dc0aae4 \ No newline at end of file +444e3a9bd1f3cd8805370a9e560b5c3720d58e778dc4ae812d9023d3b06d77fc \ No newline at end of file diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected index 55b02f242..2c6477e57 100644 --- a/tests/cachekey/project/sources/git3.expected +++ b/tests/cachekey/project/sources/git3.expected @@ -1 +1 @@ -d3c796a610f0ce13c7f4dc99ce5d0f0a8674c7bbd8c53de28248711b44d427a1 \ No newline at end of file +f2c3ab6ee644ba9507f3ddd33880fa78f92c80ff872bc1ad676e0a5c40ff3493 \ No newline at end of file diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected index fd349363e..32f292fd0 100644 --- a/tests/cachekey/project/sources/local1.expected +++ b/tests/cachekey/project/sources/local1.expected @@ -1 +1 @@ -02f863aedaf175f4b4bb9979a538bd1fb6eabd0ef9d77a633b141c072d957df5 \ No newline at end of file +bbe25b17a5190532ab36663eb08d3f855a45e09108a80ef619a1198e9d730dcf \ No newline at end of file diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected index 6aa588c9e..f0e548eef 100644 --- a/tests/cachekey/project/sources/local2.expected +++ b/tests/cachekey/project/sources/local2.expected @@ -1 +1 @@ -b9c5019375c715cb6a239714c78a1bd702d4acf81518f0c05da0217a74c365df \ No newline at end of file +36e4110f09d9a4ba06fb5c18c8633a7d9a482da28a51e44873624014a2b87438 \ No newline at end of file diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected index d3fb12439..de7d92294 100644 --- a/tests/cachekey/project/sources/ostree1.expected +++ b/tests/cachekey/project/sources/ostree1.expected @@ -1 +1 @@ -8e9f5115d54df9feff28accf061f18b77458816897fdd841d7399bc0d5375633 \ No newline at end of file +dfb4df92f5afb7fd4856db0d028ece5fc465040045f6043e8081a63683297c8a \ No newline at end of file diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected index 85cd460ec..026dbdf55 100644 --- a/tests/cachekey/project/sources/patch1.expected +++ b/tests/cachekey/project/sources/patch1.expected @@ -1 +1 @@ -52f6d22e92bb627d48f6957d0c55d57d0da5e228ea0d43d108283425b3cb13f4 \ No newline at end of file +a85bc578b9c12f0b380e064cbc835d675ac1d0d3c74f821d20c7ace2b802351f \ No newline at end of file diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected index 54636aa88..9f5b12437 100644 --- a/tests/cachekey/project/sources/patch2.expected +++ b/tests/cachekey/project/sources/patch2.expected @@ -1 +1 @@ -1937060b2df8c11f37b031751beacdd28b471324734f372b970d29ba3d605b0d \ No newline at end of file +ec64492225ef13d90236385b6d3e19aa127b8db1e33837cce180e7dd1657d95c \ No newline at end of file diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected index bc3f10148..ca1243717 100644 --- a/tests/cachekey/project/sources/patch3.expected +++ b/tests/cachekey/project/sources/patch3.expected @@ -1 +1 @@ -cdccd2228c0329b99d22703aac874b4a3a78753adb759c4fae8add533e933d2f \ No newline at end of file +d6eb497e7760976b8e9e41f3f80c0f152361188876662eb703c0ddc4984945a6 \ No newline at end of file diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected index 7fcee4c9f..05b7cda23 100644 --- a/tests/cachekey/project/sources/pip1.expected +++ b/tests/cachekey/project/sources/pip1.expected @@ -1 +1 @@ -24dcac92e44ef96217b625941d7571dd03fea4a86f34177e280ba9e3b20dea32 \ No newline at end of file +ff8b62cdb1eb65e84082e7af4c745a42e0d0259c25ab0101f42f812e873a53a8 \ No newline at end of file diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected index f9ceb1bf3..a86a0f419 100644 --- a/tests/cachekey/project/sources/remote1.expected +++ b/tests/cachekey/project/sources/remote1.expected @@ -1 +1 @@ -4eec6ce665ea7c970b4df61d2033dd5a49764e9e97855054c475c066fc29efc7 \ No newline at end of file +009d779779361f1546d808945aaa05765e7d17b6b8b7a383d658ceaeddcff0db \ No newline at end of file diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected index cc235357c..43b5ce0c8 100644 --- a/tests/cachekey/project/sources/remote2.expected +++ b/tests/cachekey/project/sources/remote2.expected @@ -1 +1 @@ -15f1af6230e981740c7103f7b8327e53a74acbbcd40579fc9990bd39ae8f049a \ No newline at end of file +d4f791a9185a4f2c18c3046c41efb26c0a833b73fc52abb3cddd4fe3ac130196 \ No newline at end of file diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected index d61f558d4..8fcec06f6 100644 --- a/tests/cachekey/project/sources/tar1.expected +++ b/tests/cachekey/project/sources/tar1.expected @@ -1 +1 @@ -3601b0102ce629bba95734d5136e4589348dde3d74402185777c2aef6ebe9c82 \ No newline at end of file +81542513d239d19a2b1ba826fb1c650cbc106e9457cbadb0ae49c9af07aeb263 \ No newline at end of file diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected index 40c46cf50..e0c85d6bd 100644 --- a/tests/cachekey/project/sources/tar2.expected +++ b/tests/cachekey/project/sources/tar2.expected @@ -1 +1 @@ -9a1230d0f68187d35bd083115e545bc153b2b06879c661ac30a73a03f876ac01 \ No newline at end of file +29f5688daecb262a4d818ec00a1bfc071fa9e54081aa098f9ef36f7e5dbdb065 \ No newline at end of file diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected index 04b82aadf..2f6a7d72c 100644 --- a/tests/cachekey/project/sources/zip1.expected +++ b/tests/cachekey/project/sources/zip1.expected @@ -1 +1 @@ -a00fbe9c48fcf071e2a84fb2e39d7939c13eac0f8b07262fb52b7746f0a6f160 \ No newline at end of file +0b8b08e15b1d2ac12d7c9c4f4f9977b9c1363e017f938f6a871516f443d985b8 \ No newline at end of file diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected index 4c3e6e71e..59fae2cce 100644 --- a/tests/cachekey/project/sources/zip2.expected +++ b/tests/cachekey/project/sources/zip2.expected @@ -1 +1 @@ -fe0e42a62eba9f4f96016eb6b0a3807401e9f9f8ded3c1ec3e500a431de0e2eb \ No newline at end of file +943e869041c5a7319d63dacf4d10b717f43863d7754f5417d6f784a2f287d1be \ No newline at end of file diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index 7d793fade..9be3be5f5 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -5accdddcdf28c48173381d9bd5d178dfe9b4b69b9ca456995d0a0d0908654275 \ No newline at end of file +1b1f2ab00c24e9c1da970922d2f80a4cee71c5e578a0c5c0b2bca24249d5e822 \ No newline at end of file -- cgit v1.2.1