diff options
author | Thiago da Silva <thiagodasilva@gmail.com> | 2019-05-03 16:52:53 -0400 |
---|---|---|
committer | Thiago da Silva <thiagodasilva@gmail.com> | 2019-05-07 15:44:00 -0400 |
commit | 37e257913c161905f96cde8e783aaa501a4e419d (patch) | |
tree | 71be149d689f5ea3a70d8344c14e46800a7c297b /docker | |
parent | ce1ba6a51d27271be9cdf9b5915b39bab5498948 (diff) | |
download | swift-37e257913c161905f96cde8e783aaa501a4e419d.tar.gz |
Add Dockerfile to build a SAIO container image
Added Dockerfile and configuration files under the
new `docker` directory to build a Swift All in One container
image.
The primary goal of this container image is to provide
application developers with a Swift/S3 endpoint that can be
tested against during the development cycle.
Added Zuul jobs to build docker container
Change-Id: Ib10975a1862a4fa8c4eeaba139837cf1f22b272a
Diffstat (limited to 'docker')
38 files changed, 759 insertions, 0 deletions
diff --git a/docker/install_scripts/00_swift_needs.sh b/docker/install_scripts/00_swift_needs.sh new file mode 100755 index 000000000..89e737f50 --- /dev/null +++ b/docker/install_scripts/00_swift_needs.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# adduser -D -H syslog && \ +for user in "swift"; do + id -u $user > /dev/null 2>&1 + if [ ! $? == '0' ]; then + adduser -D $user + printf "created user $user\n" + fi +done +printf "\n" +# mkdir /srv/node && \ +# mkdir /var/spool/rsyslog && \ +# chown -R swift:swift /srv/node/ && \ +for dirname in "/srv/node" "$HOME/bin" "/opt" "/var/cache/swift" " /var/log/socklog/swift" "/var/log/swift/" "/var/run/swift"; do + if [ ! -d $dirname ]; then + mkdir -p $dirname + printf "created $dirname\n" + fi +done +# mkdir -p $HOME/bin && \ +# mkdir -p /opt diff --git a/docker/install_scripts/10_apk_install_prereqs.sh b/docker/install_scripts/10_apk_install_prereqs.sh new file mode 100755 index 000000000..58ee435a3 --- /dev/null +++ b/docker/install_scripts/10_apk_install_prereqs.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +apk add --update \ + linux-headers \ + gnupg \ + git \ + curl \ + rsync \ + memcached \ + openssl \ + openssl-dev \ + sqlite \ + sqlite-libs \ + sqlite-dev \ + xfsprogs \ + autoconf \ + automake \ + libtool \ + make \ + zlib-dev \ + g++ \ + libffi \ + libffi-dev \ + libxslt \ + libxslt-dev \ + libxml2 \ + libxml2-dev \ + python \ + python-dev \ + py-pip diff --git a/docker/install_scripts/15_install_python_and_pip_src.sh b/docker/install_scripts/15_install_python_and_pip_src.sh new file mode 100755 index 000000000..286523db3 --- /dev/null +++ b/docker/install_scripts/15_install_python_and_pip_src.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +echo +echo +echo +echo "building python and pip" + +# export PATH=$PATH:/usr/include + +mkdir $BUILD_DIR/python27 +mkdir $BUILD_DIR/python36 + +echo +echo +echo +echo "building python 2.7.15" + +cd $BUILD_DIR/python27 +wget https://www.python.org/ftp/python/2.7.15/Python-2.7.15.tgz +tar -zxvf Python-2.7.15.tgz +cd Python-2.7.15 +./configure --enable-optimizations +make +make DESTDIR=/opt/python27 install + +echo +echo +echo +echo "building python 3.6.5" + +cd $BUILD_DIR/python36 +wget https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz +tar -zxvf Python-3.6.5.tgz +cd Python-3.6.5 +./configure --enable-optimizations +make +make DESTDIR=/opt/python36 install + +export PATH=$PATH:/opt/python27/usr/local/bin:/opt/python36/usr/local/bin +echo "export PATH=$PATH:/opt/python27/usr/local/bin:/opt/python36/usr/local/bin" >> /etc/profile + +echo +echo +echo +echo "building pip" +wget https://bootstrap.pypa.io/get-pip.py +python ./get-pip.py + +echo +echo +echo +echo "deleting python internal test dirs" +for f in `cat /opt/swift/docker/install_scripts/python_test_dirs` ; do rm -rf $f; done + +rm -rf $BUILD_DIR/python27 +rm -rf $BUILD_DIR/python36 diff --git a/docker/install_scripts/17_compile_liberasurecode.sh b/docker/install_scripts/17_compile_liberasurecode.sh new file mode 100755 index 000000000..c346d4898 --- /dev/null +++ b/docker/install_scripts/17_compile_liberasurecode.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# Get liberasurecode +cd $BUILD_DIR +git clone https://github.com/openstack/liberasurecode.git +cd liberasurecode/ +./autogen.sh +./configure +make +#make test +make install +# cp -r /usr/local/lib /usr/lib/python3.6/site-packages/ && \ +cd $BUILD_DIR +rm -rf $BUILD_DIR/liberasurecode diff --git a/docker/install_scripts/20_pip_install_prereqs.sh b/docker/install_scripts/20_pip_install_prereqs.sh new file mode 100755 index 000000000..fe41b3b36 --- /dev/null +++ b/docker/install_scripts/20_pip_install_prereqs.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +echo "+ + + + + + + + + + upgrading pip" && \ +pip install -U pip && \ +cd /opt/swift && \ +pip install -r requirements.txt + +#echo "+ + + + + + + + + + installing pastedeploy" && \ +#pip install pastedeploy && \ +#echo "+ + + + + + + + + + installing eventlet" && \ +#pip install eventlet && \ +#echo "+ + + + + + + + + + installing greenlet" && \ +#pip install greenlet && \ +#echo "+ + + + + + + + + + installing netifaces" && \ +#pip install netifaces && \ +#echo "+ + + + + + + + + + installing setuptools" && \ +#pip install setuptools && \ +#echo "+ + + + + + + + + + installing requests" && \ +#pip install requests && \ +#echo "+ + + + + + + + + + installing six" && \ +#pip install six && \ +#echo "+ + + + + + + + + + installing cryptography" && \ +#pip install cryptography && \ +#echo "+ + + + + + + + + + installing dnspython" && \ +#pip install dnspython +#echo "+ + + + + + + + + + installing xattr" && \ +#pip install xattr +#echo "+ + + + + + + + + + installing pyeclib" && \ +#pip install pyeclib +#echo "+ + + + + + + + + + installing lxml" && \ +#pip install lxml diff --git a/docker/install_scripts/50_swift_install.sh b/docker/install_scripts/50_swift_install.sh new file mode 100755 index 000000000..5a6ec2c6e --- /dev/null +++ b/docker/install_scripts/50_swift_install.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +pip install -U pip && \ +cd /opt/swift && \ +pip install -r requirements.txt && \ +pip install -e . + +cp doc/saio/bin/* $HOME/bin +chmod +x $HOME/bin/* +sed -i "s/bash/sh/g" $HOME/bin/* +sed -i "s/sudo //g" $HOME/bin/* +mkdir /root/tmp +echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.shrc +echo "export PYTHON_EGG_CACHE=/root/tmp" >> $HOME/.shrc +echo "export ENV=$HOME/.shrc" >> $HOME/.profile +chmod +x $HOME/.shrc +chmod +x $HOME/.profile diff --git a/docker/install_scripts/60_pip_uninstall_dev.sh b/docker/install_scripts/60_pip_uninstall_dev.sh new file mode 100755 index 000000000..081b4292b --- /dev/null +++ b/docker/install_scripts/60_pip_uninstall_dev.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +echo "- - - - - - - - uninstalling simplejson" +pip uninstall --yes simplejson +echo "- - - - - - - - uninstalling pyopenssl" +pip uninstall --yes pyopenssl +echo "- - - - - - - - deleting python3-dev residue (config-3.6m-x86_64-linux-gnu)" +rm -rf /opt/usr/local/lib/python3.6/config-3.6m-x86_64-linux-gnu/ diff --git a/docker/install_scripts/99_apk_uninstall_dev.sh b/docker/install_scripts/99_apk_uninstall_dev.sh new file mode 100755 index 000000000..34c9fc4fc --- /dev/null +++ b/docker/install_scripts/99_apk_uninstall_dev.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +cd / +rm -rf /build + +apk del gnupg +apk del git +apk del openssl-dev +apk del sqlite-dev +apk del autoconf +apk del automake +apk del libtool +apk del make +apk del zlib-dev +apk del g++ +apk del libffi-dev +apk del libxslt-dev +apk del libxml2-dev +apk del python-dev +rm -rf /var/cache/apk/* diff --git a/docker/install_scripts/python_test_dirs b/docker/install_scripts/python_test_dirs new file mode 100644 index 000000000..593815e8f --- /dev/null +++ b/docker/install_scripts/python_test_dirs @@ -0,0 +1,19 @@ +/opt/python/usr/local/lib/python3.6//ctypes/test +/opt/python/usr/local/lib/python3.6//distutils/tests +/opt/python/usr/local/lib/python3.6//idlelib/idle_test +/opt/python/usr/local/lib/python3.6//lib2to3/tests +/opt/python/usr/local/lib/python3.6//sqlite3/test +/opt/python/usr/local/lib/python3.6//test +/opt/python/usr/local/lib/python3.6//tkinter/test +/opt/python/usr/local/lib/python2.7/bsddb/test +/opt/python/usr/local/lib/python2.7/ctypes/test +/opt/python/usr/local/lib/python2.7/distutils/tests +/opt/python/usr/local/lib/python2.7/email/test +/opt/python/usr/local/lib/python2.7/idlelib/idle_test +/opt/python/usr/local/lib/python2.7/json/tests +/opt/python/usr/local/lib/python2.7/lib-tk/test +/opt/python/usr/local/lib/python2.7/lib2to3/tests +/opt/python/usr/local/lib/python2.7/site-packages/simplejson/tests +/opt/python/usr/local/lib/python2.7/sqlite3/test +/opt/python/usr/local/lib/python2.7/test +/opt/python/usr/local/lib/python2.7/unittest/test diff --git a/docker/rootfs/etc/cont-init.d/01_swift_logs b/docker/rootfs/etc/cont-init.d/01_swift_logs new file mode 100644 index 000000000..fcc3e4c6a --- /dev/null +++ b/docker/rootfs/etc/cont-init.d/01_swift_logs @@ -0,0 +1,4 @@ +#!/bin/sh + +s6-setuidgid swift ln -s /var/log/socklog/swift/swift_all/current /var/log/swift/all.log +s6-setuidgid swift ln -s /var/log/socklog/swift/proxy_server/current /var/log/swift/proxy_access.log diff --git a/docker/rootfs/etc/cont-init.d/02_build_remakerings b/docker/rootfs/etc/cont-init.d/02_build_remakerings new file mode 100644 index 000000000..e49d4a911 --- /dev/null +++ b/docker/rootfs/etc/cont-init.d/02_build_remakerings @@ -0,0 +1,3 @@ +#!/usr/bin/with-contenv sh + +exec s6-setuidgid swift /etc/swift_build/prepare_rings diff --git a/docker/rootfs/etc/fix-attrs.d/logging b/docker/rootfs/etc/fix-attrs.d/logging new file mode 100644 index 000000000..a8de63977 --- /dev/null +++ b/docker/rootfs/etc/fix-attrs.d/logging @@ -0,0 +1,2 @@ +/var/log/swift true swift 0755 0755 +/var/spool/rsyslog true syslog 0700 0700 diff --git a/docker/rootfs/etc/fix-attrs.d/srv_node b/docker/rootfs/etc/fix-attrs.d/srv_node new file mode 100644 index 000000000..adee7ec5e --- /dev/null +++ b/docker/rootfs/etc/fix-attrs.d/srv_node @@ -0,0 +1 @@ +/srv/node true swift 0700 0700 diff --git a/docker/rootfs/etc/fix-attrs.d/swift b/docker/rootfs/etc/fix-attrs.d/swift new file mode 100644 index 000000000..bf368314a --- /dev/null +++ b/docker/rootfs/etc/fix-attrs.d/swift @@ -0,0 +1,4 @@ +/etc/swift true swift 0700 0700 +/etc/swift/mime.types true swift 0700 0700 +/var/run/swift true swift 0755 0755 +/var/cache/swift true swift 0755 0755 diff --git a/docker/rootfs/etc/fix-attrs.d/tmp b/docker/rootfs/etc/fix-attrs.d/tmp new file mode 100644 index 000000000..8a6d27a5c --- /dev/null +++ b/docker/rootfs/etc/fix-attrs.d/tmp @@ -0,0 +1 @@ +/tmp true root 0700 0700 diff --git a/docker/rootfs/etc/profile b/docker/rootfs/etc/profile new file mode 100644 index 000000000..9ad1060e2 --- /dev/null +++ b/docker/rootfs/etc/profile @@ -0,0 +1,11 @@ +export CHARSET=UTF-8 +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/python/usr/local/bin +export PAGER=less +export PS1='\h:\w\$ ' +umask 022 + +for script in /etc/profile.d/*.sh ; do + if [ -r $script ] ; then + . $script + fi +done diff --git a/docker/rootfs/etc/rsyncd.conf b/docker/rootfs/etc/rsyncd.conf new file mode 100644 index 000000000..2d4a1fb35 --- /dev/null +++ b/docker/rootfs/etc/rsyncd.conf @@ -0,0 +1,23 @@ +uid = swift +gid = swift +log file = /var/log/rsyncd.log +pid file = /var/run/rsyncd.pid +address = 127.0.0.1 + +[account] +max connections = 2 +path = /srv/node/ +read only = false +lock file = /var/lock/account.lock + +[container] +max connections = 2 +path = /srv/node/ +read only = false +lock file = /var/lock/container.lock + +[object] +max connections = 2 +path = /srv/node/ +read only = false +lock file = /var/lock/object.lock diff --git a/docker/rootfs/etc/rsyslog.conf b/docker/rootfs/etc/rsyslog.conf new file mode 100644 index 000000000..fa42783da --- /dev/null +++ b/docker/rootfs/etc/rsyslog.conf @@ -0,0 +1,64 @@ +# /etc/rsyslog.conf Configuration file for rsyslog. +# +# For more information see +# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html +# +# Default logging rules can be found in /etc/rsyslog.d/50-default.conf + + +################# +#### MODULES #### +################# + +$ModLoad imuxsock # provides support for local system logging +#$ModLoad imklog # provides kernel logging support +#$ModLoad immark # provides --MARK-- message capability + +# provides UDP syslog reception +#$ModLoad imudp +#$UDPServerRun 514 + +# provides TCP syslog reception +#$ModLoad imtcp +#$InputTCPServerRun 514 + +# Enable non-kernel facility klog messages +$KLogPermitNonKernelFacility on + +########################### +#### GLOBAL DIRECTIVES #### +########################### + +# +# Use traditional timestamp format. +# To enable high precision timestamps, comment out the following line. +# +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# Filter duplicated messages +$RepeatedMsgReduction on + +# Disable rate-limiting of log entries +$SystemLogRateLimitInterval 0 +$SystemLogRateLimitBurst 0 + +# +# Set the default permissions for all log files. +# +$FileOwner syslog +$FileGroup adm +$FileCreateMode 0640 +$DirCreateMode 0755 +$Umask 0022 +$PrivDropToUser syslog +$PrivDropToGroup syslog + +# +# Where to place spool and state files +# +$WorkDirectory /var/spool/rsyslog + +# +# Include all config files in /etc/rsyslog.d/ +# +$IncludeConfig /etc/rsyslog.d/*.conf diff --git a/docker/rootfs/etc/rsyslog.d/00-swift.conf b/docker/rootfs/etc/rsyslog.d/00-swift.conf new file mode 100644 index 000000000..00adb5725 --- /dev/null +++ b/docker/rootfs/etc/rsyslog.d/00-swift.conf @@ -0,0 +1,16 @@ +# NOTE: we used to enable UDP logging here, but we switched +# back to just unix domain socket. + +#$imjournalRatelimitInterval 60 +#$imjournalRatelimitBurst 600000 + +# *.* @127.0.0.1:514 + +# Log all Swift proxy-server access log lines (local2) to +# /var/log/swift/proxy_access.log +local2.* /var/log/swift/proxy_access.log;RSYSLOG_FileFormat + +# Log all Swift lines to /var/log/swift/all.log +# AND PREVENT FURTHER LOGGING OF THEM (eg. to /var/log/syslog) +local0.*;local2.* /var/log/swift/all.log;RSYSLOG_TraditionalFileFormat +& ~ diff --git a/docker/rootfs/etc/rsyslog.d/50-default.conf b/docker/rootfs/etc/rsyslog.d/50-default.conf new file mode 100644 index 000000000..d8283d485 --- /dev/null +++ b/docker/rootfs/etc/rsyslog.d/50-default.conf @@ -0,0 +1,68 @@ +# Default rules for rsyslog. +# +# For more information see rsyslog.conf(5) and /etc/rsyslog.conf + +# +# First some standard log files. Log by facility. +# +auth,authpriv.* /var/log/auth.log +*.*;auth,authpriv.none -/var/log/syslog +#cron.* /var/log/cron.log +#daemon.* -/var/log/daemon.log +#kern.* -/var/log/kern.log +#lpr.* -/var/log/lpr.log +#mail.* -/var/log/mail.log +#user.* -/var/log/user.log + +# +# Logging for the mail system. Split it up so that +# it is easy to write scripts to parse these files. +# +#mail.info -/var/log/mail.info +#mail.warn -/var/log/mail.warn +mail.err /var/log/mail.err + +# +# Logging for INN news system. +# +news.crit /var/log/news/news.crit +news.err /var/log/news/news.err +news.notice -/var/log/news/news.notice + +# +# Some "catch-all" log files. +# +#*.=debug;\ +# auth,authpriv.none;\ +# news.none;mail.none -/var/log/debug +#*.=info;*.=notice;*.=warn;\ +# auth,authpriv.none;\ +# cron,daemon.none;\ +# mail,news.none -/var/log/messages + +# +# Emergencies are sent to everybody logged in. +# +*.emerg :omusrmsg:* + +# +# I like to have messages displayed on the console, but only on a virtual +# console I usually leave idle. +# +#daemon,mail.*;\ +# news.=crit;news.=err;news.=notice;\ +# *.=debug;*.=info;\ +# *.=notice;*.=warn /dev/tty8 + +# The named pipe /dev/xconsole is for the `xconsole' utility. To use it, +# you must invoke `xconsole' with the `-file' option: +# +# $ xconsole -file /dev/xconsole [...] +# +# NOTE: adjust the list below, or you'll go crazy if you have a reasonably +# busy site.. +# +daemon.*;mail.*;\ + news.err;\ + *.=debug;*.=info;\ + *.=notice;*.=warn |/dev/xconsole diff --git a/docker/rootfs/etc/services.d/memcached/run b/docker/rootfs/etc/services.d/memcached/run new file mode 100644 index 000000000..a6d5f994d --- /dev/null +++ b/docker/rootfs/etc/services.d/memcached/run @@ -0,0 +1,3 @@ +#!/usr/bin/execlineb -P + +memcached -u root -l 127.0.0.1 diff --git a/docker/rootfs/etc/services.d/swift-account/run b/docker/rootfs/etc/services.d/swift-account/run new file mode 100644 index 000000000..f31098314 --- /dev/null +++ b/docker/rootfs/etc/services.d/swift-account/run @@ -0,0 +1,5 @@ +#!/bin/sh +source /etc/profile + +# swift-account-server /etc/swift/account-server.conf +exec s6-setuidgid swift swift-init account restart --no-daemon diff --git a/docker/rootfs/etc/services.d/swift-container/run b/docker/rootfs/etc/services.d/swift-container/run new file mode 100644 index 000000000..0ed34dad0 --- /dev/null +++ b/docker/rootfs/etc/services.d/swift-container/run @@ -0,0 +1,5 @@ +#!/bin/sh +source /etc/profile + +# swift-container-server /etc/swift/container-server.conf +exec s6-setuidgid swift swift-init container restart --no-daemon diff --git a/docker/rootfs/etc/services.d/swift-object/run b/docker/rootfs/etc/services.d/swift-object/run new file mode 100644 index 000000000..80f8c8d61 --- /dev/null +++ b/docker/rootfs/etc/services.d/swift-object/run @@ -0,0 +1,4 @@ +#!/bin/sh +source /etc/profile + +exec s6-setuidgid swift swift-init object restart --no-daemon diff --git a/docker/rootfs/etc/services.d/swift-proxy/run b/docker/rootfs/etc/services.d/swift-proxy/run new file mode 100644 index 000000000..be73b5398 --- /dev/null +++ b/docker/rootfs/etc/services.d/swift-proxy/run @@ -0,0 +1,5 @@ +#!/bin/sh +source /etc/profile + +# swift-proxy-server /etc/swift/proxy-server.conf +exec s6-setuidgid swift swift-init proxy restart --no-daemon diff --git a/docker/rootfs/etc/socklog.rules/swift_account_server b/docker/rootfs/etc/socklog.rules/swift_account_server new file mode 100644 index 000000000..7a8ddb8ea --- /dev/null +++ b/docker/rootfs/etc/socklog.rules/swift_account_server @@ -0,0 +1,3 @@ +- ++\local5.* +/var/log/socklog/swift/account_server diff --git a/docker/rootfs/etc/socklog.rules/swift_all b/docker/rootfs/etc/socklog.rules/swift_all new file mode 100644 index 000000000..6ce9c899d --- /dev/null +++ b/docker/rootfs/etc/socklog.rules/swift_all @@ -0,0 +1,7 @@ +- ++\local5.* ++\local4.* ++\local3.* ++\local2.* ++\local0.* +/var/log/socklog/swift/swift_all diff --git a/docker/rootfs/etc/socklog.rules/swift_container_server b/docker/rootfs/etc/socklog.rules/swift_container_server new file mode 100644 index 000000000..428119e96 --- /dev/null +++ b/docker/rootfs/etc/socklog.rules/swift_container_server @@ -0,0 +1,3 @@ +- ++\local4.* +/var/log/socklog/swift/container_server diff --git a/docker/rootfs/etc/socklog.rules/swift_object_server b/docker/rootfs/etc/socklog.rules/swift_object_server new file mode 100644 index 000000000..e2e92fe9d --- /dev/null +++ b/docker/rootfs/etc/socklog.rules/swift_object_server @@ -0,0 +1,3 @@ +- ++\local3.* +/var/log/socklog/swift/object_server diff --git a/docker/rootfs/etc/socklog.rules/swift_proxy_server b/docker/rootfs/etc/socklog.rules/swift_proxy_server new file mode 100644 index 000000000..b74308216 --- /dev/null +++ b/docker/rootfs/etc/socklog.rules/swift_proxy_server @@ -0,0 +1,3 @@ +- ++\local2.* +/var/log/socklog/swift/proxy_server diff --git a/docker/rootfs/etc/swift/account-server.conf b/docker/rootfs/etc/swift/account-server.conf new file mode 100644 index 000000000..d8e95f73b --- /dev/null +++ b/docker/rootfs/etc/swift/account-server.conf @@ -0,0 +1,25 @@ +[DEFAULT] +devices = /srv/node/ +bind_ip = 127.0.0.1 +bind_port = 6002 +workers = 2 +mount_check = false +log_facility = LOG_LOCAL5 + +[pipeline:main] +pipeline = healthcheck recon account-server + +[app:account-server] +use = egg:swift#account + +[filter:recon] +use = egg:swift#recon + +[filter:healthcheck] +use = egg:swift#healthcheck + +[account-replicator] + +[account-auditor] + +[account-reaper] diff --git a/docker/rootfs/etc/swift/container-server.conf b/docker/rootfs/etc/swift/container-server.conf new file mode 100644 index 000000000..db99c43d3 --- /dev/null +++ b/docker/rootfs/etc/swift/container-server.conf @@ -0,0 +1,27 @@ +[DEFAULT] +devices = /srv/node/ +bind_ip = 127.0.0.1 +bind_port = 6001 +workers = 2 +mount_check = false +log_facility = LOG_LOCAL4 + +[pipeline:main] +pipeline = healthcheck recon container-server + +[app:container-server] +use = egg:swift#container + +[filter:recon] +use = egg:swift#recon + +[filter:healthcheck] +use = egg:swift#healthcheck + +[container-replicator] + +[container-updater] + +[container-auditor] + +[container-sync] diff --git a/docker/rootfs/etc/swift/object-server.conf b/docker/rootfs/etc/swift/object-server.conf new file mode 100644 index 000000000..3dc66aa61 --- /dev/null +++ b/docker/rootfs/etc/swift/object-server.conf @@ -0,0 +1,26 @@ +[DEFAULT] +devices = /srv/node/ +bind_ip = 127.0.0.1 +bind_port = 6000 +workers = 2 +mount_check = false +log_facility = LOG_LOCAL3 + +[pipeline:main] +pipeline = healthcheck recon object-server + +[app:object-server] +use = egg:swift#object + +[filter:recon] +use = egg:swift#recon + +[filter:healthcheck] +use = egg:swift#healthcheck + + +[object-replicator] + +[object-updater] + +[object-auditor] diff --git a/docker/rootfs/etc/swift/proxy-server.conf b/docker/rootfs/etc/swift/proxy-server.conf new file mode 100644 index 000000000..56eb8b0b2 --- /dev/null +++ b/docker/rootfs/etc/swift/proxy-server.conf @@ -0,0 +1,105 @@ +[DEFAULT] +bind_ip = 0.0.0.0 +bind_port = 8080 +log_address = /dev/log +log_facility = LOG_LOCAL2 +log_headers = false +log_level = DEBUG +log_name = proxy-server +user = swift + +[pipeline:main] +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats bulk tempurl ratelimit tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server + +[filter:catch_errors] +use = egg:swift#catch_errors + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:proxy-logging] +use = egg:swift#proxy_logging + +[filter:bulk] +use = egg:swift#bulk + +[filter:ratelimit] +use = egg:swift#ratelimit + +[filter:crossdomain] +use = egg:swift#crossdomain + +[filter:dlo] +use = egg:swift#dlo + +[filter:slo] +use = egg:swift#slo + +[filter:container_sync] +use = egg:swift#container_sync +current = //saio/saio_endpoint + +[filter:s3api] +use = egg:swift#s3api +# Add this filter to your pipeline if you want to test with the S3 protocol. +s3_acl = yes +dns_compliant_bucket_names = no +check_bucket_owner = yes + +[filter:tempurl] +use = egg:swift#tempurl + +[filter:tempauth] +use = egg:swift#tempauth +user_admin_admin = admin .admin .reseller_admin +user_test_tester = testing .admin +user_test_tester2 = testing2 .admin +user_test_tester3 = testing3 +user_test2_tester2 = testing2 .admin + +[filter:staticweb] +use = egg:swift#staticweb + +[filter:account-quotas] +use = egg:swift#account_quotas + +[filter:container-quotas] +use = egg:swift#container_quotas + +[filter:cache] +use = egg:swift#memcache + +[filter:gatekeeper] +use = egg:swift#gatekeeper + +[filter:versioned_writes] +use = egg:swift#versioned_writes +allow_versioned_writes = true + +[filter:copy] +use = egg:swift#copy + +[filter:listing_formats] +use = egg:swift#listing_formats + +[filter:symlink] +use = egg:swift#symlink + +# To enable, add the s3api middleware to the pipeline before tempauth +[filter:s3api] +use = egg:swift#s3api + +# Example to create root secret: `openssl rand -base64 32` +[filter:keymaster] +use = egg:swift#keymaster +encryption_root_secret = changeme/changeme/changeme/changeme/change/= + +# To enable use of encryption add both middlewares to pipeline, example: +# <other middleware> keymaster encryption proxy-logging proxy-server +[filter:encryption] +use = egg:swift#encryption + +[app:proxy-server] +use = egg:swift#proxy +allow_account_management = true +account_autocreate = true diff --git a/docker/rootfs/etc/swift/swift.conf b/docker/rootfs/etc/swift/swift.conf new file mode 100644 index 000000000..129a5d98a --- /dev/null +++ b/docker/rootfs/etc/swift/swift.conf @@ -0,0 +1,17 @@ +[swift-hash] +# random unique strings that can never change (DO NOT LOSE) +swift_hash_path_prefix = bd08f643f5663c4ec607 +swift_hash_path_suffix = f423bf7ab663888fe832 + +[storage-policy:0] +name = 1replica +default = true +policy_type = replication + +# [storage-policy:1] +# name = EC42 +# policy_type = erasure_coding +# ec_type = liberasurecode_rs_vand +# ec_num_data_fragments = 4 +# ec_num_parity_fragments = 2 +# ec_object_segment_size = 1048576 diff --git a/docker/rootfs/etc/swift_build/build_devices b/docker/rootfs/etc/swift_build/build_devices new file mode 100755 index 000000000..90f1c9e6a --- /dev/null +++ b/docker/rootfs/etc/swift_build/build_devices @@ -0,0 +1,62 @@ +#!/usr/bin/with-contenv sh + +cd /etc/swift +DEV_SIZE="1GB" +# POLICIES="object container account" +MY_STORAGE_TYPE=${STORAGE_TYPE:-"internal_dirs"} +MY_DEVICE_COUNT=${DEVICE_COUNT:-6} + +echo "[[ checking --privileged ]]" +ip link add dummy0 type dummy >/dev/null +if [[ $? -eq 0 ]]; then + PRIVILEGED=true + # clean the dummy0 link + ip link delete dummy0 >/dev/null +else + PRIVILEGED=false +fi + +echo "storage type is $MY_STORAGE_TYPE. container is privileged? $PRIVILEGED" + +echo "[[ checking what to use as storage devices ]]" +DEVICE_LIST="" +if [[ $MY_STORAGE_TYPE == "external_devices" ]]; then + DEVICE_LIST=$(ls /dev/ | grep -i "swift-d") + MY_DEVICE_COUNT=$(wc -w $DEVICE_LIST) + echo " using external device. devices found: $DEVICE_LIST" +elif [[ $MY_DEVICE_COUNT -le 0 ]]; then + echo "Device count must be greater than 0" + exit -1 +else + for i in $(seq 0 $(( MY_DEVICE_COUNT-1 ))); do + DEVICE_LIST="$DEVICE_LIST swift-d$i" + done + # echo " using internal devices. devices to create: $DEVICE_LIST" +fi + +if [[ $MY_STORAGE_TYPE == "internal_devices" ]]; then + for device in $DEVICE_LIST; do + truncate -s $DEV_SIZE /dev/$device; + echo " created storage device /dev/swift-d$i of $DEV_SIZE"; + done +fi + +export PATH=$PATH:/opt/python/usr/local/bin/ + +echo "[[ creating directories ]]" +for dir in $DEVICE_LIST; do + mkdir -p /srv/node/$dir; + echo " created /srv/node/$dir"; +done + +if [[ $MY_STORAGE_TYPE == "internal_devices" ]] || [[ $MY_STORAGE_TYPE == "external_devices" ]]; then + echo "[[ formating and mounting storage devices ]] " + for device in $DEVICE_LIST; do + # truncate -s $DEV_SIZE /dev/swift-d$i; + # echo "created storage device /dev/swift-d$i of $DEV_SIZE"; + mkfs.xfs -f -L D$i -i size=512 /dev/$device; + echo " created XFS file system on device /dev/$device"; + mount -t xfs -o noatime,nodiratime,logbufs=8 /dev/$device /srv/node/$device; + echo " mounted /dev/$device as /srv/node/$device"; + done +fi diff --git a/docker/rootfs/etc/swift_build/build_remakerings b/docker/rootfs/etc/swift_build/build_remakerings new file mode 100755 index 000000000..da1c842db --- /dev/null +++ b/docker/rootfs/etc/swift_build/build_remakerings @@ -0,0 +1,36 @@ +#!/usr/bin/with-contenv sh + +POLICIES="object container account" + +for p in $POLICIES; do + echo "swift-ring-builder $p.builder create 10 1 1" > /etc/swift/remakerings.$p; + echo "started /etc/swift/remakerings.$p with 'swift-ring-build create'" +done + + +for drive in `ls /srv/node/ | grep 'swift-d'`; do + echo "swift-ring-builder object.builder add r1z1-127.0.0.1:6000/$drive 1" >> /etc/swift/remakerings.object + echo "pushed command to add r1z1-127.0.0.1:6000/$drive to /etc/swift/remakerings.object" + echo "swift-ring-builder container.builder add r1z1-127.0.0.1:6001/$drive 1" >> /etc/swift/remakerings.container + echo "pushed command to add r1z1-127.0.0.1:6001/$drive to /etc/swift/remakerings.container" + echo "swift-ring-builder account.builder add r1z1-127.0.0.1:6002/$drive 1" >> /etc/swift/remakerings.account + echo "pushed command to add r1z1-127.0.0.1:6002/$drive to /etc/swift/remakerings.account" +done + +for p in $POLICIES; do + echo "swift-ring-builder $p.builder rebalance" >> /etc/swift/remakerings.$p; + echo "pushed command to rebalance ring into /etc/swift/remakerings.$p" +done + +echo "rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz" > /etc/swift/remakerings +echo "created umbrella /etc/swift/remakerings, with deleting all ring files" + +for p in $POLICIES; do + cat /etc/swift/remakerings.$p >> /etc/swift/remakerings; + echo "pushed /etc/swift/remakerings.$p to /etc/swift/remakerings" + rm -f /etc/swift/remakerings.$p; + echo "deleted /etc/swift/remakerings.$p" +done + +chmod +x /etc/swift/remakerings +echo "made remaketings executable (+x)" diff --git a/docker/rootfs/etc/swift_build/prepare_rings b/docker/rootfs/etc/swift_build/prepare_rings new file mode 100755 index 000000000..7b951ff22 --- /dev/null +++ b/docker/rootfs/etc/swift_build/prepare_rings @@ -0,0 +1,6 @@ +#!/usr/bin/with-contenv sh + +/etc/swift_build/build_devices +/etc/swift_build/build_remakerings +cd /etc/swift +/etc/swift/remakerings |