summaryrefslogtreecommitdiff
path: root/nss/automation/taskcluster/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'nss/automation/taskcluster/scripts')
-rwxr-xr-xnss/automation/taskcluster/scripts/build.sh13
-rwxr-xr-xnss/automation/taskcluster/scripts/build_gyp.sh11
-rwxr-xr-xnss/automation/taskcluster/scripts/build_nspr.sh18
-rwxr-xr-xnss/automation/taskcluster/scripts/build_nss.sh39
-rwxr-xr-xnss/automation/taskcluster/scripts/build_softoken.sh30
-rwxr-xr-xnss/automation/taskcluster/scripts/build_util.sh25
-rwxr-xr-xnss/automation/taskcluster/scripts/extend_task_graph.sh7
-rwxr-xr-xnss/automation/taskcluster/scripts/fuzz.sh26
-rwxr-xr-xnss/automation/taskcluster/scripts/gen_certs.sh12
-rwxr-xr-xnss/automation/taskcluster/scripts/run_clang_format.sh77
-rwxr-xr-xnss/automation/taskcluster/scripts/run_scan_build.sh9
-rwxr-xr-xnss/automation/taskcluster/scripts/run_tests.sh10
-rw-r--r--nss/automation/taskcluster/scripts/split.sh152
-rw-r--r--nss/automation/taskcluster/scripts/tools.sh10
14 files changed, 348 insertions, 91 deletions
diff --git a/nss/automation/taskcluster/scripts/build.sh b/nss/automation/taskcluster/scripts/build.sh
index 69968b1..649fdaa 100755
--- a/nss/automation/taskcluster/scripts/build.sh
+++ b/nss/automation/taskcluster/scripts/build.sh
@@ -1,14 +1,17 @@
#!/usr/bin/env bash
-source $(dirname $0)/tools.sh
+source $(dirname "$0")/tools.sh
-if [[ $(id -u) -eq 0 ]]; then
- # Drop privileges by re-running this script.
- exec su worker $0
+if [ -n "$NSS_BUILD_MODULAR" ]; then
+ $(dirname "$0")/build_nspr.sh || exit $?
+ $(dirname "$0")/build_util.sh || exit $?
+ $(dirname "$0")/build_softoken.sh || exit $?
+ $(dirname "$0")/build_nss.sh || exit $?
+ exit
fi
# Clone NSPR if needed.
-hg_clone https://hg.mozilla.org/projects/nspr nspr default
+hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
# Build.
make -C nss nss_build_all
diff --git a/nss/automation/taskcluster/scripts/build_gyp.sh b/nss/automation/taskcluster/scripts/build_gyp.sh
index 590e634..7190bd5 100755
--- a/nss/automation/taskcluster/scripts/build_gyp.sh
+++ b/nss/automation/taskcluster/scripts/build_gyp.sh
@@ -1,17 +1,12 @@
#!/usr/bin/env bash
-source $(dirname $0)/tools.sh
-
-if [[ $(id -u) -eq 0 ]]; then
- # Drop privileges by re-running this script.
- exec su worker -c "$0 $*"
-fi
+source $(dirname "$0")/tools.sh
# Clone NSPR if needed.
-hg_clone https://hg.mozilla.org/projects/nspr nspr default
+hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
# Build.
-nss/build.sh ${*--g -v}
+nss/build.sh -g -v "$@"
# Package.
mkdir artifacts
diff --git a/nss/automation/taskcluster/scripts/build_nspr.sh b/nss/automation/taskcluster/scripts/build_nspr.sh
new file mode 100755
index 0000000..4d19034
--- /dev/null
+++ b/nss/automation/taskcluster/scripts/build_nspr.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+
+# Clone NSPR if needed.
+hg_clone https://hg.mozilla.org/projects/nspr nspr default
+
+# Build.
+rm -rf dist
+make -C nss build_nspr
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-nspr
+mv dist dist-nspr
+tar cvfjh artifacts/dist-nspr.tar.bz2 dist-nspr
diff --git a/nss/automation/taskcluster/scripts/build_nss.sh b/nss/automation/taskcluster/scripts/build_nss.sh
new file mode 100755
index 0000000..b909bc3
--- /dev/null
+++ b/nss/automation/taskcluster/scripts/build_nss.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+source $(dirname $0)/split.sh
+
+test -d dist-softoken || { echo "run build_softoken.sh first" 1>&2; exit 1; }
+
+rm -rf nss-nss
+split_nss nss nss-nss
+
+# Build.
+export NSS_BUILD_WITHOUT_SOFTOKEN=1
+export NSS_USE_SYSTEM_FREEBL=1
+
+platform=`make -s -C nss platform`
+
+export NSPR_LIB_DIR="$PWD/dist-nspr/$platform/lib"
+export NSSUTIL_LIB_DIR="$PWD/dist-util/$platform/lib"
+export FREEBL_LIB_DIR="$PWD/dist-softoken/$platform/lib"
+export SOFTOKEN_LIB_DIR="$PWD/dist-softoken/$platform/lib"
+export FREEBL_LIBS=-lfreebl
+
+export NSS_NO_PKCS11_BYPASS=1
+export FREEBL_NO_DEPEND=1
+
+export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib:$PWD/dist-softoken/$platform/lib"
+export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH"
+export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss -I$PWD/dist-softoken/public/nss"
+
+rm -rf dist
+make -C nss-nss nss_build_all
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-nss
+mv dist dist-nss
+tar cvfjh artifacts/dist-nss.tar.bz2 dist-nss
diff --git a/nss/automation/taskcluster/scripts/build_softoken.sh b/nss/automation/taskcluster/scripts/build_softoken.sh
new file mode 100755
index 0000000..e5aaecc
--- /dev/null
+++ b/nss/automation/taskcluster/scripts/build_softoken.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+source $(dirname $0)/split.sh
+
+test -d dist-util || { echo "run build_util.sh first" 1>&2; exit 1; }
+
+rm -rf nss-softoken
+split_softoken nss nss-softoken
+
+# Build.
+platform=`make -s -C nss platform`
+export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib"
+export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH"
+export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss"
+export NSS_BUILD_SOFTOKEN_ONLY=1
+
+rm -rf dist
+make -C nss-softoken nss_build_all
+
+mv dist/private/nss/blapi.h dist/public/nss
+mv dist/private/nss/alghmac.h dist/public/nss
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-softoken
+mv dist dist-softoken
+tar cvfjh artifacts/dist-softoken.tar.bz2 dist-softoken
diff --git a/nss/automation/taskcluster/scripts/build_util.sh b/nss/automation/taskcluster/scripts/build_util.sh
new file mode 100755
index 0000000..0d2ecc5
--- /dev/null
+++ b/nss/automation/taskcluster/scripts/build_util.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+source $(dirname $0)/split.sh
+
+rm -rf nss-util
+split_util nss nss-util
+
+# Build.
+platform=`make -s -C nss platform`
+export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib"
+export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH"
+export INCLUDES="-I$PWD/dist-nspr/$platform/include"
+export NSS_BUILD_UTIL_ONLY=1
+
+rm -rf dist
+make -C nss-util nss_build_all
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-util
+mv dist dist-util
+tar cvfjh artifacts/dist-util.tar.bz2 dist-util
diff --git a/nss/automation/taskcluster/scripts/extend_task_graph.sh b/nss/automation/taskcluster/scripts/extend_task_graph.sh
index 5a3fb8d..ade84cd 100755
--- a/nss/automation/taskcluster/scripts/extend_task_graph.sh
+++ b/nss/automation/taskcluster/scripts/extend_task_graph.sh
@@ -1,11 +1,6 @@
#!/usr/bin/env bash
-set -v -e -x
-
-if [ $(id -u) = 0 ]; then
- # Drop privileges by re-running this script.
- exec su worker $0
-fi
+source $(dirname "$0")/tools.sh
mkdir -p /home/worker/artifacts
diff --git a/nss/automation/taskcluster/scripts/fuzz.sh b/nss/automation/taskcluster/scripts/fuzz.sh
index 5f8dd7b..75851ff 100755
--- a/nss/automation/taskcluster/scripts/fuzz.sh
+++ b/nss/automation/taskcluster/scripts/fuzz.sh
@@ -1,20 +1,32 @@
#!/usr/bin/env bash
-source $(dirname $0)/tools.sh
+source $(dirname "$0")/tools.sh
-if [ $(id -u) = 0 ]; then
- # Drop privileges by re-running this script.
- exec su worker -c "$0 $*"
-fi
+type="$1"
+shift
# Fetch artifact if needed.
fetch_dist
# Clone corpus.
-./nss/fuzz/clone_corpus.sh
+./nss/fuzz/config/clone_corpus.sh
+
+# Ensure we have a corpus.
+if [ ! -d "nss/fuzz/corpus/$type" ]; then
+ mkdir -p nss/fuzz/corpus/$type
+
+ set +x
+
+ # Create a corpus out of what we have.
+ for f in $(find nss/fuzz/corpus -type f); do
+ cp $f "nss/fuzz/corpus/$type"
+ done
+
+ set -x
+fi
# Fetch objdir name.
objdir=$(cat dist/latest)
# Run nssfuzz.
-LD_LIBRARY_PATH=$LD_LIBRARY_PATH:dist/$objdir/lib dist/$objdir/bin/nssfuzz $*
+dist/$objdir/bin/nssfuzz-"$type" "$@"
diff --git a/nss/automation/taskcluster/scripts/gen_certs.sh b/nss/automation/taskcluster/scripts/gen_certs.sh
index aee1001..b8d4f60 100755
--- a/nss/automation/taskcluster/scripts/gen_certs.sh
+++ b/nss/automation/taskcluster/scripts/gen_certs.sh
@@ -1,16 +1,6 @@
#!/usr/bin/env bash
-set -v -e -x
-
-source $(dirname $0)/tools.sh
-
-if [ $(id -u) = 0 ]; then
- # Stupid Docker.
- echo "127.0.0.1 localhost.localdomain" >> /etc/hosts
-
- # Drop privileges by re-running this script.
- exec su worker $0
-fi
+source $(dirname "$0")/tools.sh
# Fetch artifact if needed.
fetch_dist
diff --git a/nss/automation/taskcluster/scripts/run_clang_format.sh b/nss/automation/taskcluster/scripts/run_clang_format.sh
index c4b6029..05e6317 100755
--- a/nss/automation/taskcluster/scripts/run_clang_format.sh
+++ b/nss/automation/taskcluster/scripts/run_clang_format.sh
@@ -1,56 +1,57 @@
#!/usr/bin/env bash
-set -v -e -x
+source $(dirname "$0")/tools.sh
-if [ $(id -u) -eq 0 ]; then
- # Drop privileges by re-running this script.
- exec su worker $0 "$@"
-fi
+set +x
# Apply clang-format on the provided folder and verify that this doesn't change any file.
# If any file differs after formatting, the script eventually exits with 1.
# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
-# Includes a default set of directories.
+# Includes a default set of directories NOT to clang-format on.
+blacklist=(
+ "./automation" \
+ "./coreconf" \
+ "./doc" \
+ "./pkg" \
+ "./tests" \
+ "./lib/libpkix" \
+ "./lib/zlib" \
+ "./lib/sqlite" \
+ "./gtests/google_test" \
+ "./.hg" \
+)
+
+top="$PWD/$(dirname $0)/../../.."
+cd "$top"
if [ $# -gt 0 ]; then
dirs=("$@")
else
- top=$(dirname $0)/../../..
- dirs=( \
- "$top/cmd" \
- "$top/fuzz" \
- "$top/lib/base" \
- "$top/lib/certdb" \
- "$top/lib/certhigh" \
- "$top/lib/ckfw" \
- "$top/lib/crmf" \
- "$top/lib/cryptohi" \
- "$top/lib/dbm" \
- "$top/lib/dev" \
- "$top/lib/freebl" \
- "$top/lib/jar" \
- "$top/lib/nss" \
- "$top/lib/pk11wrap" \
- "$top/lib/pkcs7" \
- "$top/lib/pkcs12" \
- "$top/lib/pki" \
- "$top/lib/smime" \
- "$top/lib/softoken" \
- "$top/lib/ssl" \
- "$top/lib/sysinit" \
- "$top/lib/util" \
- "$top/gtests/common" \
- "$top/gtests/der_gtest" \
- "$top/gtests/freebl_gtest" \
- "$top/gtests/pk11_gtest" \
- "$top/gtests/ssl_gtest" \
- "$top/gtests/util_gtest" \
- )
+ dirs=($(find . ! -path . \( ! -regex '.*/' \) -maxdepth 2 -mindepth 1 -type d))
fi
+format_folder()
+{
+ for black in "${blacklist[@]}"; do
+ if [[ "$1" == "$black"* ]]; then
+ echo "skip $1"
+ return 1
+ fi
+ done
+ return 0
+}
+
for dir in "${dirs[@]}"; do
- find "$dir" -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
+ if format_folder "$dir" ; then
+ c="${dir//[^\/]}"
+ echo "formatting $dir ..."
+ depth=""
+ if [ "${#c}" == "1" ]; then
+ depth="-maxdepth 1"
+ fi
+ find "$dir" $depth -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
+ fi
done
TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX)
diff --git a/nss/automation/taskcluster/scripts/run_scan_build.sh b/nss/automation/taskcluster/scripts/run_scan_build.sh
index 99f80ab..4024c22 100755
--- a/nss/automation/taskcluster/scripts/run_scan_build.sh
+++ b/nss/automation/taskcluster/scripts/run_scan_build.sh
@@ -1,15 +1,10 @@
#!/usr/bin/env bash
-source $(dirname $0)/tools.sh
-
-if [ $(id -u) = 0 ]; then
- # Drop privileges by re-running this script.
- exec su worker $0 $@
-fi
+source $(dirname "$0")/tools.sh
# Clone NSPR if needed.
if [ ! -d "nspr" ]; then
- hg_clone https://hg.mozilla.org/projects/nspr nspr default
+ hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
fi
# Build.
diff --git a/nss/automation/taskcluster/scripts/run_tests.sh b/nss/automation/taskcluster/scripts/run_tests.sh
index 4c87e7e..b8e2676 100755
--- a/nss/automation/taskcluster/scripts/run_tests.sh
+++ b/nss/automation/taskcluster/scripts/run_tests.sh
@@ -1,14 +1,6 @@
#!/usr/bin/env bash
-source $(dirname $0)/tools.sh
-
-if [ $(id -u) = 0 ]; then
- # Stupid Docker.
- echo "127.0.0.1 localhost.localdomain" >> /etc/hosts
-
- # Drop privileges by re-running this script.
- exec su worker $0
-fi
+source $(dirname "$0")/tools.sh
# Fetch artifact if needed.
fetch_dist
diff --git a/nss/automation/taskcluster/scripts/split.sh b/nss/automation/taskcluster/scripts/split.sh
new file mode 100644
index 0000000..c43c3e4
--- /dev/null
+++ b/nss/automation/taskcluster/scripts/split.sh
@@ -0,0 +1,152 @@
+copy_top()
+{
+ srcdir_="$1"
+ dstdir_="$2"
+ files=`find "$srcdir_" -maxdepth 1 -mindepth 1 -type f`
+ for f in $files; do
+ cp -p "$f" "$dstdir_"
+ done
+}
+
+split_util() {
+ nssdir="$1"
+ dstdir="$2"
+
+ # Prepare a source tree only containing files to build nss-util:
+ #
+ # nss/dbm full directory
+ # nss/coreconf full directory
+ # nss top files only
+ # nss/lib top files only
+ # nss/lib/util full directory
+
+ # Copy everything.
+ cp -R $nssdir $dstdir
+
+ # Skip gtests when building.
+ sed '/^DIRS = /s/ gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
+
+ # Remove subdirectories that we don't want.
+ rm -rf $dstdir/cmd
+ rm -rf $dstdir/tests
+ rm -rf $dstdir/lib
+ rm -rf $dstdir/automation
+ rm -rf $dstdir/gtests
+ rm -rf $dstdir/doc
+
+ # Start with an empty cmd lib directories to be filled selectively.
+ mkdir $dstdir/cmd
+ cp $nssdir/cmd/Makefile $dstdir/cmd
+ cp $nssdir/cmd/manifest.mn $dstdir/cmd
+ cp $nssdir/cmd/platlibs.mk $dstdir/cmd
+ cp $nssdir/cmd/platrules.mk $dstdir/cmd
+
+ # Copy some files at the top and the util subdirectory recursively.
+ mkdir $dstdir/lib
+ cp $nssdir/lib/Makefile $dstdir/lib
+ cp $nssdir/lib/manifest.mn $dstdir/lib
+ cp -R $nssdir/lib/util $dstdir/lib/util
+}
+
+split_softoken() {
+ nssdir="$1"
+ dstdir="$2"
+
+ # Prepare a source tree only containing files to build nss-softoken:
+ #
+ # nss/dbm full directory
+ # nss/coreconf full directory
+ # nss top files only
+ # nss/lib top files only
+ # nss/lib/freebl full directory
+ # nss/lib/softoken full directory
+ # nss/lib/softoken/dbm full directory
+
+ # Copy everything.
+ cp -R $nssdir $dstdir
+
+ # Skip gtests when building.
+ sed '/^DIRS = /s/ gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
+
+ # Remove subdirectories that we don't want.
+ rm -rf $dstdir/cmd
+ rm -rf $dstdir/tests
+ rm -rf $dstdir/lib
+ rm -rf $dstdir/pkg
+ rm -rf $dstdir/automation
+ rm -rf $dstdir/gtests
+ rm -rf $dstdir/doc
+
+ # Start with an empty lib directory and copy only what we need.
+ mkdir $dstdir/lib
+ copy_top $nssdir/lib $dstdir/lib
+ cp -R $nssdir/lib/dbm $dstdir/lib/dbm
+ cp -R $nssdir/lib/freebl $dstdir/lib/freebl
+ cp -R $nssdir/lib/softoken $dstdir/lib/softoken
+ cp -R $nssdir/lib/sqlite $dstdir/lib/sqlite
+
+ mkdir $dstdir/cmd
+ copy_top $nssdir/cmd $dstdir/cmd
+ cp -R $nssdir/cmd/bltest $dstdir/cmd/bltest
+ cp -R $nssdir/cmd/ecperf $dstdir/cmd/ecperf
+ cp -R $nssdir/cmd/fbectest $dstdir/cmd/fbectest
+ cp -R $nssdir/cmd/fipstest $dstdir/cmd/fipstest
+ cp -R $nssdir/cmd/lib $dstdir/cmd/lib
+ cp -R $nssdir/cmd/lowhashtest $dstdir/cmd/lowhashtest
+ cp -R $nssdir/cmd/shlibsign $dstdir/cmd/shlibsign
+
+ mkdir $dstdir/tests
+ copy_top $nssdir/tests $dstdir/tests
+
+ cp -R $nssdir/tests/cipher $dstdir/tests/cipher
+ cp -R $nssdir/tests/common $dstdir/tests/common
+ cp -R $nssdir/tests/ec $dstdir/tests/ec
+ cp -R $nssdir/tests/lowhash $dstdir/tests/lowhash
+
+ cp $nssdir/lib/util/verref.h $dstdir/lib/freebl
+ cp $nssdir/lib/util/verref.h $dstdir/lib/softoken
+ cp $nssdir/lib/util/verref.h $dstdir/lib/softoken/legacydb
+}
+
+split_nss() {
+ nssdir="$1"
+ dstdir="$2"
+
+ # Prepare a source tree only containing files to build nss:
+ #
+ # nss/dbm full directory
+ # nss/coreconf full directory
+ # nss top files only
+ # nss/lib top files only
+ # nss/lib/freebl full directory
+ # nss/lib/softoken full directory
+ # nss/lib/softoken/dbm full directory
+
+ # Copy everything.
+ cp -R $nssdir $dstdir
+
+ # Remove subdirectories that we don't want.
+ rm -rf $dstdir/lib/freebl
+ rm -rf $dstdir/lib/softoken
+ rm -rf $dstdir/lib/util
+ rm -rf $dstdir/cmd/bltest
+ rm -rf $dstdir/cmd/fipstest
+ rm -rf $dstdir/cmd/rsaperf_low
+
+ # Copy these headers until the upstream bug is accepted
+ # Upstream https://bugzilla.mozilla.org/show_bug.cgi?id=820207
+ cp $nssdir/lib/softoken/lowkeyi.h $dstdir/cmd/rsaperf
+ cp $nssdir/lib/softoken/lowkeyti.h $dstdir/cmd/rsaperf
+
+ # Copy verref.h which will be needed later during the build phase.
+ cp $nssdir/lib/util/verref.h $dstdir/lib/ckfw/builtins/verref.h
+ cp $nssdir/lib/util/verref.h $dstdir/lib/nss/verref.h
+ cp $nssdir/lib/util/verref.h $dstdir/lib/smime/verref.h
+ cp $nssdir/lib/util/verref.h $dstdir/lib/ssl/verref.h
+ cp $nssdir/lib/util/templates.c $dstdir/lib/nss/templates.c
+
+ # FIXME: Skip util_gtest because it links with libnssutil.a. Note
+ # that we can't use libnssutil3.so instead, because util_gtest
+ # depends on internal symbols not exported from the shared library.
+ sed '/ util_gtest \\/d' $dstdir/gtests/manifest.mn > $dstdir/gtests/manifest.mn-t && mv $dstdir/gtests/manifest.mn-t $dstdir/gtests/manifest.mn
+}
diff --git a/nss/automation/taskcluster/scripts/tools.sh b/nss/automation/taskcluster/scripts/tools.sh
index dacfdeb..46d567e 100644
--- a/nss/automation/taskcluster/scripts/tools.sh
+++ b/nss/automation/taskcluster/scripts/tools.sh
@@ -2,11 +2,21 @@
set -v -e -x
+if [[ $(id -u) -eq 0 ]]; then
+ # Drop privileges by re-running this script.
+ # Note: this mangles arguments, better to avoid running scripts as root.
+ exec su worker -c "$0 $*"
+fi
+
# Usage: hg_clone repo dir [revision=@]
hg_clone() {
repo=$1
dir=$2
rev=${3:-@}
+ if [ -d "$dir" ]; then
+ hg pull -R "$dir" -ur "$rev" "$repo" && return
+ rm -rf "$dir"
+ fi
for i in 0 2 5; do
sleep $i
hg clone -r "$rev" "$repo" "$dir" && return
View Lorry Controller Status