diff options
Diffstat (limited to 'google-startup-scripts/usr/share/google')
14 files changed, 0 insertions, 1262 deletions
diff --git a/google-startup-scripts/usr/share/google/boto/boot_setup.py b/google-startup-scripts/usr/share/google/boto/boot_setup.py deleted file mode 100755 index e9f3924..0000000 --- a/google-startup-scripts/usr/share/google/boto/boot_setup.py +++ /dev/null @@ -1,92 +0,0 @@ -#! /usr/bin/python -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A simple start up script to set up the system boto.cfg file. - -This will hit the metadata server to get the appropriate project id -and install the compute authenication plugin. - -Note that this starts with whatever is in /etc/boto.cfg.template, adds -to that and then persists it into /etc/boto.cfg. This is done so that -the system boto.cfg can be removed prior to image packaging. -""" - -from ConfigParser import SafeConfigParser -import os -import sys -import textwrap -import urllib2 - -NUMERIC_PROJECT_ID_URL=('http://169.254.169.254/' - 'computeMetadata/v1/project/numeric-project-id') -SYSTEM_BOTO_CONFIG_TEMPLATE='/etc/boto.cfg.template' -SYSTEM_BOTO_CONFIG='/etc/boto.cfg' -AUTH_PLUGIN_DIR='/usr/share/google/boto/boto_plugins' - - -def GetNumericProjectId(): - """Get the numeric project ID for this VM.""" - try: - request = urllib2.Request(NUMERIC_PROJECT_ID_URL) - request.add_unredirected_header('Metadata-Flavor', 'Google') - return urllib2.urlopen(request).read() - except (urllib2.URLError, urllib2.HTTPError, IOError), e: - return None - - -def AddConfigFileHeader(fp): - s = ("""\ - This file is automatically created at boot time by the %s script. - Do not edit this file directly. If you need to add items to this - file, create/edit %s instead and then re-run the script.""" - % (os.path.abspath(__file__), SYSTEM_BOTO_CONFIG_TEMPLATE)) - fp.write('\n'.join(['# ' + s for s in textwrap.wrap(textwrap.dedent(s), - break_on_hyphens=False)])) - fp.write('\n\n') - - -def main(argv): - config = SafeConfigParser() - config.read(SYSTEM_BOTO_CONFIG_TEMPLATE) - - # TODO(user): Figure out if we need a retry here. - project_id = GetNumericProjectId() - if not project_id: - # Our project doesn't support service accounts. - return - - if not config.has_section('GSUtil'): - config.add_section('GSUtil') - config.set('GSUtil', 'default_project_id', project_id) - config.set('GSUtil', 'default_api_version', '2') - - if not config.has_section('GoogleCompute'): - config.add_section('GoogleCompute') - # TODO(user): Plumb a metadata value to set this. We probably want - # to namespace the metadata values in some way like - # 'boto_auth.servicee_account'. - config.set('GoogleCompute', 'service_account', 'default') - - if not config.has_section('Plugin'): - config.add_section('Plugin') - config.set('Plugin', 'plugin_directory', AUTH_PLUGIN_DIR) - - with open(SYSTEM_BOTO_CONFIG, 'w') as configfile: - AddConfigFileHeader(configfile) - config.write(configfile) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/google-startup-scripts/usr/share/google/boto/boto_plugins/compute_auth.py b/google-startup-scripts/usr/share/google/boto/boto_plugins/compute_auth.py deleted file mode 100644 index 97d3e20..0000000 --- a/google-startup-scripts/usr/share/google/boto/boto_plugins/compute_auth.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Authentication module for using Google Compute service accounts.""" - -import json -import urllib2 - -from boto.auth_handler import AuthHandler -from boto.auth_handler import NotReadyToAuthenticate - -META_DATA_SERVER_BASE_URL=( - 'http://169.254.169.254/computeMetadata/v1') - -SERVICE_ACCOUNT_SCOPES_URL=(META_DATA_SERVER_BASE_URL + - '/instance/service-accounts/%s/scopes?alt=json') -SERVICE_ACCOUNT_TOKEN_URL=(META_DATA_SERVER_BASE_URL + - '/instance/service-accounts/%s/token?alt=json') - -GS_SCOPES = set([ - 'https://www.googleapis.com/auth/devstorage.read_only', - 'https://www.googleapis.com/auth/devstorage.read_write', - 'https://www.googleapis.com/auth/devstorage.full_control', - ]) - - -class ComputeAuth(AuthHandler): - """Google Compute service account auth handler. - - What happens is that the boto library reads the system config file - (/etc/boto.cfg) and looks at a config value called 'plugin_directory'. It - then loads the python files in that and find classes derived from - boto.auth_handler.AuthHandler. - """ - - capability = ['google-oauth2', 's3'] - - def __init__(self, path, config, provider): - self.service_account = config.get('GoogleCompute', 'service_account', '') - if provider.name == 'google' and self.service_account: - self.scopes = self.__GetGSScopes() - if not self.scopes: - raise NotReadyToAuthenticate() - else: - raise NotReadyToAuthenticate() - - def __GetJSONMetadataValue(self, url): - try: - request = urllib2.Request(url) - request.add_unredirected_header('Metadata-Flavor', 'Google') - data = urllib2.urlopen(request).read() - return json.loads(data) - except (urllib2.URLError, urllib2.HTTPError, IOError): - return None - - def __GetGSScopes(self): - """Return all Google Storage scopes available on this VM.""" - scopes = self.__GetJSONMetadataValue( - SERVICE_ACCOUNT_SCOPES_URL % self.service_account) - if scopes: - return list(GS_SCOPES.intersection(set(scopes))) - return None - - def __GetAccessToken(self): - """Return an oauth2 access token for Google Storage.""" - token_info = self.__GetJSONMetadataValue( - SERVICE_ACCOUNT_TOKEN_URL % self.service_account) - if token_info: - return token_info['access_token'] - return None - - def add_auth(self, http_request): - http_request.headers['Authorization'] = ( - 'OAuth %s' % self.__GetAccessToken()) diff --git a/google-startup-scripts/usr/share/google/fetch_script b/google-startup-scripts/usr/share/google/fetch_script deleted file mode 100755 index 72ba9ac..0000000 --- a/google-startup-scripts/usr/share/google/fetch_script +++ /dev/null @@ -1,148 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Fetch a script from metadata and store it in the /var/run directory. -declare -r LOGFILE=/var/log/google.log - -if [[ -x /usr/bin/logger ]]; then - declare -r LOGGER=/usr/bin/logger -else - declare -r LOGGER=/bin/logger -fi - -declare -r CURL_RETRY_LIMIT=10 -declare -r CURL_TIMEOUT=10 - -function log() { - echo "$@" | ${LOGGER} -t google -p daemon.info - echo "$@" >> ${LOGFILE} -} - -function download_url_with_logfile() { - local readonly url=$1 - local readonly dest=$2 - local readonly logfile=$3 - - if [[ "$url" =~ gs://* ]]; then - log "Downloading url from ${url} to ${dest} using gsutil" - gsutil cp "${url}" "${dest}" > "${logfile}" 2>&1 && return 0 - log "Failed to download $url" - return 1 - fi - - # Many of the Google Storage URLs are supported below. - # It is prefered that customers specify their object using - # its gs://<bucket>/<object> url. - - bucket="[a-z0-9][-_.a-z0-9]*[a-z0-9]" - - # Accept any non-empty string that doesn't contain a wildcard character - # gsutil interprets some characters as wildcards - # These characters in object names make it difficult or impossible - # to perform various wildcard operations using gsutil - # For a complete list use "gsutil help naming" - object="[^\*\?][^\*\?]*" - - # For all validation tests: - # alphanumeric ranges should only include ascii characters - export LC_COLLATE=C - - # Check for the Google Storage URLs: - # http://<bucket>.storage.googleapis.com/<object>/ - # https://<bucket>.storage.googleapis.com/<object>/ - if [[ "$url" =~ http[s]?://${bucket}\.storage\.googleapis\.com/${object} ]]; then - log "Downloading url from ${url} to ${dest} using gsutil" - # Create a url that can be interpreted by gsutil - gsurl=$(echo "$url" | sed "s/^https\?:\/\/\($bucket\)\.storage\.googleapis\.com\/\($object\)$/gs:\/\/\1\/\2/") - gsutil cp ${gsurl} ${dest} 2> ${logfile} && return 0 - # Check for the other possible Google Storage URLS: - # http://storage.googleapis.com/<bucket>/<object>/ - # https://storage.googleapis.com/<bucket>/<object>/ - # - # The following are deprecated but checked - # http://commondatastorage.googleapis.com/<bucket>/<object>/ - # https://commondatastorage.googleapis.com/<bucket>/<object>/ - elif [[ "$url" =~ http[s]?://(commondata)?storage\.googleapis\.com/${bucket}/${object} ]]; then - log "Downloading url from ${url} to ${dest} using gsutil" - # Create a url that can be interpreted by gsutil - gsurl=$(echo "$url" | sed "s/^https\?:\/\/\(commondata\|\)storage\.googleapis\.com\/\($bucket\)\/\($object\)$/gs:\/\/\2\/\3/") - gsutil cp "${gsurl}" "${dest}" 2> "${logfile}" && return 0 - else - log "URL ${url} is not located in Google Storage" - fi - - # Unauthenticated download of the object. - log "Downloading url from ${url} to ${dest} using curl" - curl --max-time "${CURL_TIMEOUT}" --retry "${CURL_RETRY_LIMIT}" \ - 2>> "${logfile}" -o "${dest}" -L -- "${url}" && return 0; - - log "Failed to download $url" - return 1 -} - -function download_url() { - local readonly url=$1 - local readonly dest=$2 - local readonly logfile=$(mktemp) - download_url_with_logfile "${url}" "${dest}" "${logfile}" - return_code=$? - # If the script was unable to download then report to the syslog. - if [[ "${return_code}" != "0" ]]; then - log "$(<"${logfile}")" - else - rm -f "${logfile}" - fi - return "${return_code}" -} - -function get_metadata_attribute() { - local readonly varname=$1 - /usr/share/google/get_metadata_value "attributes/${varname}" - return $? -} - -function fetch_script() { - # Try to use the script-url, then the script metadata. - # Check the script url first. - script=$1 - script_type=$2 - url_type="${script_type}-script" - url="${url_type}-url" - - local readonly script_url="$(get_metadata_attribute ${url})" - if [[ -n "${script_url}" ]]; then - log "${url} metadata flag: ${script_url}" - download_url "${script_url}" "${script}" - if [[ $? != 0 ]]; then - log "Could not download ${script_type} script ${script_url}." - else - log "Successfully downloaded ${script_type} script ${script_url}." - fi - else - local readonly metadata_script="$(get_metadata_attribute ${url_type})" - if [[ -n "${metadata_script}" ]]; then - echo "${metadata_script}" > "${script}" - log "${script_type} script found in metadata." - else - log $(curl "http://metadata.google.internal/computeMetadata/v1/instance/?recursive=True" -H "Metadata-Flavor: Google") - log "No ${script_type} script found in metadata." - fi - fi - [[ -e "${script}" ]] && chmod 700 "${script}" - - return 0 -} - -fetch_script "$1" "$2" diff --git a/google-startup-scripts/usr/share/google/first-boot b/google-startup-scripts/usr/share/google/first-boot deleted file mode 100755 index b346b65..0000000 --- a/google-startup-scripts/usr/share/google/first-boot +++ /dev/null @@ -1,94 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run initialization code the first time this image boots on a given instance. - -declare -r INSTANCE_FILE=${PREFIX}/var/lib/google/vm-instance-id -declare -r LOCK_FILE=${INSTANCE_FILE}.lock - -mkdir -p ${PREFIX}/var/lib/google/per-instance - -function log() { - if [[ -x ${PREFIX}/usr/bin/logger ]]; then - echo $* | ${PREFIX}/usr/bin/logger -t first-boot -p auth.info - else - echo $* >&2 - fi -} - -function get_instance_id() { - ${PREFIX}/usr/share/google/get_metadata_value id 2>/dev/null -} - -# Checks the instance id has changed. -# Exits with return code 0 if the instance id hasn't changed. -function check_stored_instance_id() { - local readonly instance_id=$1 - - if [[ "${instance_id}" == "" ]]; then - # Cannot determine instance id. Either we're not running on a Compute VM, - # or networking hasn't started up yet, etc. - exit 1 - fi - - if [[ "${instance_id}" != "unknown-instance" && - "${instance_id}" == "$(cat ${INSTANCE_FILE} 2>/dev/null)" ]]; then - # Instance id is same as on disk. - exit 1 - fi -} - -# Performs host key setup if the instance has changed. -# Otherwise we exit with a non-zero return code. -function manage_stored_instance_id() { - local readonly instance_id=$1 - - # Create a subshell to manage the lock file. The file lock is released - # when the subshell exits. - ( - # Open LOCK_FILE on FD 200 and lock it. This prevents concurrent calls - # to regenerate host keys that spam console output. - flock -e 200 - - # Checks whether the instance has changed. - # If the instance hasn't changed, exit the script. - check_stored_instance_id ${instance_id} - - # If the instance hasn't changed, we have now exited the subshell. - # Since the instance changed, we do host key regeneration. - log "Running first-boot" - - # Regenerate host keys for ssh. - if [[ -x ${PREFIX}/usr/share/google/regenerate-host-keys ]]; then - ${PREFIX}/usr/share/google/regenerate-host-keys - fi - - # We are booting this instance for the first time. - echo ${instance_id} > ${INSTANCE_FILE} - ) 200> ${LOCK_FILE} - - return $? -} - -declare -r INSTANCE_ID=$(get_instance_id) - -manage_stored_instance_id ${INSTANCE_ID} -if [[ $? != 0 ]]; then - # The instance hasn't changed so exit. - exit 0 -fi - -# Make a per-instance data directory. -mkdir -p ${PREFIX}/var/lib/google/per-instance/${INSTANCE_ID} diff --git a/google-startup-scripts/usr/share/google/get_metadata_value b/google-startup-scripts/usr/share/google/get_metadata_value deleted file mode 100755 index c4e0eb6..0000000 --- a/google-startup-scripts/usr/share/google/get_metadata_value +++ /dev/null @@ -1,73 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Get a metadata value from the metadata server. -declare -r VARNAME=$1 -declare -r MDS_PREFIX=http://metadata.google.internal/computeMetadata/v1 -declare -r MDS_TRIES=${MDS_TRIES:-100} - -function print_metadata_value() { - local readonly tmpfile=$(mktemp) - http_code=$(curl -f "${1}" -H "Metadata-Flavor: Google" -w "%{http_code}" \ - -s -o ${tmpfile} 2>/dev/null) - local readonly return_code=$? - # If the command completed successfully, print the metadata value to stdout. - if [[ ${return_code} == 0 && ${http_code} == 200 ]]; then - cat ${tmpfile} - fi - rm -f ${tmpfile} - return ${return_code} -} - -function print_metadata_value_if_exists() { - local return_code=1 - local readonly url=$1 - print_metadata_value ${url} - return_code=$? - return ${return_code} -} - -function get_metadata_value() { - local readonly varname=$1 - # Print the instance metadata value. - print_metadata_value_if_exists ${MDS_PREFIX}/instance/${varname} - return_code=$? - # If the instance doesn't have the value, try the project. - if [[ ${return_code} != 0 ]]; then - print_metadata_value_if_exists ${MDS_PREFIX}/project/${varname} - return_code=$? - fi - return ${return_code} -} - -function get_metadata_value_with_retries() { - local return_code=1 # General error code. - for ((count=0; count <= ${MDS_TRIES}; count++)); do - get_metadata_value $VARNAME - return_code=$? - case $return_code in - # No error. We're done. - 0) exit ${return_code};; - # Failed to resolve host or connect to host. Retry. - 6|7) sleep 0.3; continue;; - # A genuine error. Exit. - *) exit ${return_code}; - esac - done - # Exit with the last return code we got. - exit ${return_code} -} - -get_metadata_value_with_retries diff --git a/google-startup-scripts/usr/share/google/onboot b/google-startup-scripts/usr/share/google/onboot deleted file mode 100755 index 482d384..0000000 --- a/google-startup-scripts/usr/share/google/onboot +++ /dev/null @@ -1,162 +0,0 @@ -#!/bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prep the image for Google Compute services. -# -# Do NOT "set -e" - -# Exit out early if we've run before. -declare -r RUNFILE=/var/run/google.onboot -if [ -f ${RUNFILE} ]; then - exit 0 -fi - -PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin - -declare -r LOGFILE=/var/log/google.log - -if [ -x /usr/bin/logger ]; then - declare -r LOGGER=/usr/bin/logger -else - declare -r LOGGER=/bin/logger -fi - -declare -r BOTO_SETUP_SCRIPT=/usr/share/google/boto/boot_setup.py -declare -r GOOGLE_ENVIRONMENT=/var/run/google.environment - -function log() { - echo $* | ${LOGGER} -t google -p daemon.info - echo $* >> ${LOGFILE} -} - -function set_interrupts() { - if [[ -x /usr/share/google/set-interrupts ]]; then - /usr/share/google/set-interrupts - fi -} - -function virtionet_irq_affinity() { - if [[ -x /usr/share/google/virtionet-irq-affinity ]]; then - /usr/share/google/virtionet-irq-affinity - fi -} - -function first_boot() { - if [[ -x /usr/share/google/first-boot ]]; then - /usr/share/google/first-boot - fi -} - -function get_metadata_value() { - local readonly varname=$1 - /usr/share/google/get_metadata_value ${varname} - return $? -} - -function do_environment() { - echo "INSTANCE_ID=$(get_metadata_value id)" > ${GOOGLE_ENVIRONMENT} -} - -function do_init() { - log "onboot initializing" - - do_environment - - # If it exists, run the boto bootstrap script. This will set things - # up so that gsutil will just work with any provisioned service - # account. - if [ -x ${BOTO_SETUP_SCRIPT} ]; then - log "Running Boto setup script at ${BOTO_SETUP_SCRIPT}" - ${BOTO_SETUP_SCRIPT} >> ${LOGFILE} 2>&1 - fi - - return 0 -} - -function print_ssh_key_fingerprints() { - log "SSH public key fingerprints" - - if [ -e /etc/ssh/ssh_host_rsa_key.pub ]; then - log "RSA public key" - ssh-keygen -lf /etc/ssh/ssh_host_rsa_key.pub - else - log "No RSA public key found." - fi - - if [ -e /etc/ssh/ssh_host_dsa_key.pub ]; then - log "DSA public key" - ssh-keygen -lf /etc/ssh/ssh_host_dsa_key.pub - else - log "No DSA public key found." - fi - - if [ -e /etc/ssh/ssh_host_ecdsa_key.pub ]; then - log "ECDSA public key" - ssh-keygen -lf /etc/ssh/ssh_host_ecdsa_key.pub - else - log "No ECDSA public key found." - fi - - return 0 -} - -function check_for_connection() { - local count=0 - local return_code=0 - - log "Checking for metadata server connection." - while true; do - ((count++)) - MDS_TRIES=1 /usr/share/google/get_metadata_value "?recursive=True" - return_code=$? - case ${return_code} in - # No error. Connection is active. - 0) break;; - # Failed to resolve host or connect to host. Retry indefinitely. - 6|7) sleep 1.0 - log "Waiting for metadata server, attempt ${count}" - # After 7 minutes, add a console message denoting a probable network - # issue. On systems using dhclient there is an attempt to obtain an IP - # for 60 seconds followed by a 5 minute wait period. After 7 minutes, - # this cycle will have run through twice. After this period of time, it - # is not known when a DHCP lease might be obtained and the network - # interface fully operational. - if ((count >= 7*60+1)); then - log "There is likely a problem with the network." - fi - continue;; - # A genuine error but a connection exists. - *) - log "Check for connection non-fatal error getting metadata ${return_code}" - break;; - esac - done - # Return the last return code we got. - return ${return_code} -} - -set_interrupts -virtionet_irq_affinity -check_for_connection -first_boot -do_init -print_ssh_key_fingerprints - -if [ -x /sbin/initctl ]; then - /sbin/initctl emit --no-wait google-onboot-has-run -fi - -# Indicate that we've run already. -touch ${RUNFILE} diff --git a/google-startup-scripts/usr/share/google/regenerate-host-keys b/google-startup-scripts/usr/share/google/regenerate-host-keys deleted file mode 100755 index fb9d7fd..0000000 --- a/google-startup-scripts/usr/share/google/regenerate-host-keys +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Regenerates the SSH host keys when the VM is restarted with a new IP -# address. Booting a VM from an image with a known SSH key allows a -# number of attacks, so this script regenerates the host key whenever -# the IP address changes. (This applies on firstboot, but also if the -# VM disk has been used for another image.) - -log() { - logger -t regenerate-host-keys -p auth.info -s "$@" -} - -sshd_cmd() { - local cmd=$1 - log "${cmd}ing sshd" - if [[ -x /etc/init.d/ssh || -f /etc/init/ssh.conf ]]; then - service ssh ${cmd} - fi - if [[ -x /etc/init.d/sshd || -f /etc/init/sshd.conf ]]; then - service sshd ${cmd} - fi -} - -generate_key() { - local key_type=$1 - local key_dest=$2 - local tmp_dir=$(mktemp -d /tmp/keystore.XXXXXXXX) - local tmp_file="/${tmp_dir}/keyfile.$$"; - local log_file=$(mktemp); - log "Regenerating sshd key ${key_dest}" - ssh-keygen -N '' -t ${key_type} -f ${tmp_file} > ${log_file} 2>&1 - if [[ $? == 0 ]]; then - rm -f ${key_dest} ${key_dest}.pub - cp -f ${tmp_file} ${key_dest} - cp -f ${tmp_file}.pub ${key_dest}.pub - else - log "Could not create sshd key ${key_dest}" - log "$(cat ${log_file})" - fi - rm -rf ${tmp_dir} - rm -f ${log_file} -} - -regenerate_host_keys() { - log "Regenerating SSH Host Keys for: $new_ip_address (previously $old_ip_address)." - rm -f /etc/ssh/ssh_host_key /etc/ssh_host_key.pub # SSH1 RSA key. - for key_file in /etc/ssh/ssh_host_*_key; do - # Parse out the type of key, matching the * in the for loop command above. - key_type=$(basename "${key_file}" _key) - key_type=${key_type#ssh_host_} - - generate_key "${key_type}" "${key_file}" - done - # Allow sshd to come up if we were suppressing it. - if [[ $(cat /etc/ssh/sshd_not_to_be_run 2>/dev/null) == "GOOGLE" ]]; then - rm -f /etc/ssh/sshd_not_to_be_run - fi - if [[ -x /bin/systemctl ]]; then - exit - else - # Start sshd if it was not running. - sshd_cmd start - # Reload sshd config if it already was running. - sshd_cmd reload - fi -} - -regenerate_host_keys diff --git a/google-startup-scripts/usr/share/google/run-scripts b/google-startup-scripts/usr/share/google/run-scripts deleted file mode 100755 index 46af979..0000000 --- a/google-startup-scripts/usr/share/google/run-scripts +++ /dev/null @@ -1,54 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run startup scripts that should happen "Late" at boot. -# Run shutdown scripts that should happen as soon as the instances -# begin to power down. -# -# Do NOT "set -e" -declare -r SCRIPT=$1 -declare -r SCRIPT_TYPE=$2 - -if [ -x /usr/bin/logger ]; then - declare -r LOGGER=/usr/bin/logger -else - declare -r LOGGER=/bin/logger -fi - -LOG_CMD="${LOGGER} -t ${SCRIPT_TYPE}script -p daemon.info" - -function log() { - echo "$@" | ${LOG_CMD} -} - -declare -r GOOGLE_ENVIRONMENT=/var/run/google.environment - -function copy_and_run() { - local source=$1 - local dest=$(mktemp) - cat "${source}" >> "${dest}" - chmod u+x "${dest}" - log "Running ${SCRIPT_TYPE} script ${source}" - "${dest}" 2>&1 | ${LOG_CMD} - log "Finished running ${SCRIPT_TYPE} script ${source}" - rm -f "${dest}" -} - -if [[ -e "${SCRIPT}" ]]; then - ( - [ -r ${GOOGLE_ENVIRONMENT} ] && source ${GOOGLE_ENVIRONMENT}; - copy_and_run "${SCRIPT}" - ) -fi diff --git a/google-startup-scripts/usr/share/google/run-shutdown-scripts b/google-startup-scripts/usr/share/google/run-shutdown-scripts deleted file mode 100755 index 61377e9..0000000 --- a/google-startup-scripts/usr/share/google/run-shutdown-scripts +++ /dev/null @@ -1,31 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run shutdown scripts that should happen as soon as the instances -# begin to power down. -# -# Do NOT "set -e" -PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin - -declare -r SHUTDOWN_SCRIPT=/var/run/google.shutdown.script - -# NOTE -# Make sure that the shutdown script completes within 90 seconds, so -# that the OS has time to complete its shutdown, including flushing -# buffers to disk. -# -# The shutdown script blocks other shutdown operations from proceeding. -/usr/share/google/fetch_script ${SHUTDOWN_SCRIPT} shutdown -/usr/share/google/run-scripts ${SHUTDOWN_SCRIPT} shutdown diff --git a/google-startup-scripts/usr/share/google/run-startup-scripts b/google-startup-scripts/usr/share/google/run-startup-scripts deleted file mode 100755 index b9e2667..0000000 --- a/google-startup-scripts/usr/share/google/run-startup-scripts +++ /dev/null @@ -1,27 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run startup scripts that should happen "Late" at boot. -# -# Do NOT "set -e" -PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin - -declare -r STARTUP_SCRIPT=/var/run/google.startup.script - -# Make sure all udev changes settle before running startup scripts. -udevadm settle - -/usr/share/google/fetch_script ${STARTUP_SCRIPT} startup -/usr/share/google/run-scripts ${STARTUP_SCRIPT} startup diff --git a/google-startup-scripts/usr/share/google/safe_format_and_mount b/google-startup-scripts/usr/share/google/safe_format_and_mount deleted file mode 100755 index 8e68037..0000000 --- a/google-startup-scripts/usr/share/google/safe_format_and_mount +++ /dev/null @@ -1,152 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Mount a disk, formatting it if necessary. If the disk looks like it may -# have been formatted before, we will not format it. -# -# This script uses blkid and file to search for magic "formatted" bytes -# at the beginning of the disk. Furthermore, it attempts to use fsck to -# repair the filesystem before formatting it. - -FSCK=fsck.ext4 -MOUNT_OPTIONS="discard,defaults" -MKFS="mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 -F" -if [ -f /etc/redhat-release ]; then - if grep -q '6\..' /etc/redhat-release; then - # lazy_journal_init is not recognized in redhat 6 - MKFS="mkfs.ext4 -E lazy_itable_init=0 -F" - elif grep -q '7\..' /etc/redhat-release; then - FSCK=fsck.xfs - MKFS=mkfs.xfs - fi -fi - -LOGTAG=safe_format_and_mount -LOGFACILITY=user - -function log() { - local readonly severity=$1; shift; - logger -t ${LOGTAG} -p ${LOGFACILITY}.${severity} -s "$@" -} - -function log_command() { - local readonly log_file=$(mktemp) - local readonly retcode - log info "Running: $*" - $* > ${log_file} 2>&1 - retcode=$? - # only return the last 1000 lines of the logfile, just in case it's HUGE. - tail -1000 ${log_file} | logger -t ${LOGTAG} -p ${LOGFACILITY}.info -s - rm -f ${log_file} - return ${retcode} -} - -function help() { - cat >&2 <<EOF -$0 [-f fsck_cmd] [-m mkfs_cmd] [-o mount_opts] <device> <mountpoint> -EOF - exit 0 -} - -while getopts ":hf:o:m:" opt; do - case $opt in - h) help;; - f) FSCK=$OPTARG;; - o) MOUNT_OPTIONS=$OPTARG;; - m) MKFS=$OPTARG;; - -) break;; - \?) log error "Invalid option: -${OPTARG}"; exit 1;; - :) log "Option -${OPTARG} requires an argument."; exit 1;; - esac -done - -shift $(($OPTIND - 1)) -readonly DISK=$1 -readonly MOUNTPOINT=$2 - -[[ -z ${DISK} ]] && help -[[ -z ${MOUNTPOINT} ]] && help - -function disk_looks_unformatted() { - blkid ${DISK} - if [[ $? == 0 ]]; then - return 0 - fi - - local readonly file_type=$(file --special-files ${DISK}) - case ${file_type} in - *filesystem*) - return 0;; - esac - - return 1 -} - -function format_disk() { - log_command ${MKFS} ${DISK} -} - -function try_repair_disk() { - log_command ${FSCK} -a ${DISK} - local readonly fsck_return=$? - if [[ ${fsck_return} -ge 8 ]]; then - log error "Fsck could not correct errors on ${DISK}" - return 1 - fi - if [[ ${fsck_return} -gt 0 ]]; then - log warning "Fsck corrected errors on ${DISK}" - fi - return 0 -} - -function try_mount() { - local mount_retcode - try_repair_disk - - log_command mount -o ${MOUNT_OPTIONS} ${DISK} ${MOUNTPOINT} - mount_retcode=$? - if [[ ${mount_retcode} == 0 ]]; then - return 0 - fi - - # Check to see if it looks like a filesystem before formatting it. - disk_looks_unformatted ${DISK} - if [[ $? == 0 ]]; then - log error "Disk ${DISK} looks formatted but won't mount. Giving up." - return ${mount_retcode} - fi - - # The disk looks like it's not been formatted before. - format_disk - if [[ $? != 0 ]]; then - log error "Format of ${DISK} failed." - fi - - log_command mount -o ${MOUNT_OPTIONS} ${DISK} ${MOUNTPOINT} - mount_retcode=$? - if [[ ${mount_retcode} == 0 ]]; then - return 0 - fi - log error "Tried everything we could, but could not mount ${DISK}." - return ${mount_retcode} -} - -log warn "=====================================================================" -log warn "WARNING: safe_format_and_mount is deprecated." -log warn "See https://cloud.google.com/compute/docs/disks/persistent-disks" -log warn "for additional instructions." -log warn "=====================================================================" -try_mount -exit $? diff --git a/google-startup-scripts/usr/share/google/set-hostname b/google-startup-scripts/usr/share/google/set-hostname deleted file mode 100755 index 9b71e4d..0000000 --- a/google-startup-scripts/usr/share/google/set-hostname +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Deal with a new hostname assignment. - -if [ -n "$new_host_name" ] && [ -n "$new_ip_address" ]; then - # Delete entries with new_host_name or new_ip_address in /etc/hosts. - sed -i '/Added by Google/d' /etc/hosts - - # Add an entry for our new_host_name/new_ip_address in /etc/hosts. - echo "${new_ip_address} ${new_host_name} ${new_host_name%%.*} # Added by Google" >> /etc/hosts -fi - -# /sbin/dhclient-scripts in both ubuntu and centos have some problems for us: -# 1) BOUND doesn't always set hostname (e.g. if old_host_name is unset in -# precise pangolin) -# 2) Using too long of a FQDN as a hostname causes some tools to break in -# some distros (e.g. ssh-keygen) and hostname tool complains when given -# a FQDN that is > 64 bytes. -# -# As a result, we set the host name in all circumstances here, to the truncated -# unqualified domain name. - -if [ -n "$new_host_name" ]; then - hostname ${new_host_name%%.*} - - # Let syslogd know we've changed the hostname. - pkill -HUP syslogd -fi diff --git a/google-startup-scripts/usr/share/google/set-interrupts b/google-startup-scripts/usr/share/google/set-interrupts deleted file mode 100755 index 36ccaec..0000000 --- a/google-startup-scripts/usr/share/google/set-interrupts +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -total_cpus=`nproc` - -config_nvme() -{ - current_cpu=0 - for dev in /sys/bus/pci/drivers/nvme/* - do - if [ ! -d $dev ] - then - continue - fi - for irq_info in $dev/msi_irqs/* - do - if [ ! -f $irq_info ] - then - continue - fi - current_cpu=$((current_cpu % total_cpus)) - cpu_mask=`printf "%x" $((1<<current_cpu))` - irq=$(basename $irq_info)$a - echo Setting IRQ $irq smp_affinity to $cpu_mask - echo $cpu_mask > /proc/irq/$irq/smp_affinity - current_cpu=$((current_cpu+1)) - done - done -} - -config_scsi() -{ - irqs=() - for device in /sys/bus/virtio/drivers/virtio_scsi/virtio* - do - ssd=0 - for target_path in $device/host*/target*/* - do - if [ ! -f $target_path/model ] - then - continue - fi - model=$(cat $target_path/model) - if [[ $model =~ .*EphemeralDisk.* ]] - then - ssd=1 - for queue_path in $target_path/block/sd*/queue - do - echo noop > $queue_path/scheduler - echo 0 > $queue_path/add_random - echo 512 > $queue_path/nr_requests - echo 0 > $queue_path/rotational - echo 0 > $queue_path/rq_affinity - echo 1 > $queue_path/nomerges - done - fi - done - if [[ $ssd == 1 ]] - then - request_queue=$(basename $device)-request - irq=$(cat /proc/interrupts |grep $request_queue| awk '{print $1}'| sed 's/://') - irqs+=($irq) - fi - done - irq_count=${#irqs[@]} - if [ $irq_count != 0 ] - then - stride=$((total_cpus / irq_count)) - stride=$((stride < 1 ? 1 : stride)) - current_cpu=0 - for irq in "${irqs[@]}" - do - current_cpu=$(($current_cpu % $total_cpus)) - cpu_mask=`printf "%x" $((1<<$current_cpu))` - echo Setting IRQ $irq smp_affinity to $cpu_mask - echo $cpu_mask > /proc/irq/$irq/smp_affinity - current_cpu=$((current_cpu+stride)) - done - fi -} - -config_nvme -config_scsi diff --git a/google-startup-scripts/usr/share/google/virtionet-irq-affinity b/google-startup-scripts/usr/share/google/virtionet-irq-affinity deleted file mode 100755 index 6b86ee2..0000000 --- a/google-startup-scripts/usr/share/google/virtionet-irq-affinity +++ /dev/null @@ -1,141 +0,0 @@ -#! /bin/bash -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For a single-queue / no MSI-X virtionet device, sets the IRQ affinities to -# processor 0. For this virtionet configuration, distributing IRQs to all -# processors results in comparatively high cpu utilization and comparatively -# low network bandwidth. -# -# For a multi-queue / MSI-X virtionet device, sets the IRQ affinities to the -# per-IRQ affinity hint. The virtionet driver maps each virtionet TX (RX) queue -# MSI-X interrupt to a unique single CPU if the number of TX (RX) queues equals -# the number of online CPUs. The mapping of network MSI-X interrupt vector to -# CPUs is stored in the virtionet MSI-X interrupt vector affinity hint. This -# configuration allows network traffic to be spread across the CPUs, giving -# each CPU a dedicated TX and RX network queue, while ensuring that all packets -# from a single flow are delivered to the same CPU. - -function log() { - if [[ -x ${PREFIX}/usr/bin/logger ]]; then - echo $* | ${PREFIX}/usr/bin/logger -t virtionet-irq-affinity -p daemon.info - else - echo $* >&2 - fi -} - -function is_decimal_int() { - [ "${1}" -eq "${1}" ] > /dev/null 2>&1 -} - -function set_channels() { - ethtool -L "${1}" combined "${2}" > /dev/null 2>&1 -} - -log "Running $(basename $0)" -NET_DEVS=/sys/bus/virtio/drivers/virtio_net/virtio* - -# Loop through all the virtionet devices and enable multi-queue -if [ -x /sbin/ethtool ]; then - for dev in $NET_DEVS; do - ETH_DEVS=${dev}/net/* - for eth_dev in $ETH_DEVS; do - eth_dev=$(basename "$eth_dev") - if ! errormsg=$(ethtool -l "$eth_dev" 2>&1); then - log "/sbin/ethtool says that $eth_dev does not support virtionet multiqueue: $errormsg" - continue - fi - num_max_channels=$(ethtool -l "$eth_dev" | grep -m 1 Combined | cut -f2) - [ "${num_max_channels}" -eq "1" ] && continue - if is_decimal_int "$num_max_channels" && \ - set_channels "$eth_dev" "$num_max_channels"; then - log "Set channels for $eth_dev to $num_max_channels" - else - log "Could not set channels for $eth_dev to $num_max_channels" - fi - done - done -else - log "/sbin/ethtool not found: cannot configure virtionet multiqueue" -fi - -for dev in $NET_DEVS -do - dev=$(basename "$dev") - irq_dir=/proc/irq/* - for irq in $irq_dir - do - smp_affinity="${irq}/smp_affinity" - [ ! -f "${smp_affinity}" ] && continue - # Classify this IRQ as virtionet intx, virtionet MSI-X, or non-virtionet - # If the IRQ type is virtionet intx, a subdirectory with the same name as - # the device will be present. If the IRQ type is virtionet MSI-X, then - # a subdirectory of the form <device name>-<input|output>.N will exist. - # In this case, N is the input (output) queue number, and is specified as - # a decimal integer ranging from 0 to K - 1 where K is the number of - # input (output) queues in the virtionet device. - virtionet_intx_dir="${irq}/${dev}" - virtionet_msix_dir_regex=".*/${dev}-(input|output)\.[0-9]+$" - if [ -d "${virtionet_intx_dir}" ]; then - # All virtionet intx IRQs are delivered to CPU 0 - log "Setting ${smp_affinity} to 01 for device ${dev}" - echo "01" > ${smp_affinity} - continue - fi - # Not virtionet intx, probe for MSI-X - virtionet_msix_found=0 - for entry in ${irq}/${dev}*; do - if [[ "$entry" =~ ${virtionet_msix_dir_regex} ]]; then - virtionet_msix_found=1 - fi - done - affinity_hint="${irq}/affinity_hint" - [ "$virtionet_msix_found" -eq 0 -o ! -f "${affinity_hint}" ] && continue - - # The affinity hint file contains a CPU mask, consisting of - # groups of up to 8 hexadecimal digits, separated by a comma. Each bit - # position in the CPU mask hex value specifies whether this interrupt - # should be delivered to the corresponding CPU. For example, if bits 0 - # and 3 are set in the affinity hint CPU mask hex value, then the - # interrupt should be delivered to CPUs 0 and 3. The virtionet device - # driver should set only a single bit in the affinity hint per MSI-X - # interrupt, ensuring each TX (RX) queue is used only by a single CPU. - # The virtionet driver will only specify an affinity hint if the number of - # TX (RX) queues equals the number of online CPUs. If no affinity hint is - # specified for an IRQ, the affinity hint file will contain all zeros. - affinity_cpumask=$(cat "${affinity_hint}") - affinity_hint_enabled=0 - # Parse the affinity hint, skip if mask is invalid or is empty (all-zeros) - OIFS=${IFS} - IFS="," - for cpu_bitmap in ${affinity_cpumask}; do - bitmap_val=$(printf "%d" "0x${cpu_bitmap}" 2>/dev/null) - if [ "$?" -ne 0 ]; then - log "Invalid affinity hint ${affinity_hint}: ${affinity_cpumask}" - affinity_hint_enabled=0 - break - elif [ "${bitmap_val}" -ne 0 ]; then - affinity_hint_enabled=1 - fi - done - IFS=${OIFS} - if [ "${affinity_hint_enabled}" -eq 0 ]; then - log "Cannot set IRQ affinity ${smp_affinity}, affinity hint disabled" - else - # Set the IRQ CPU affinity to the virtionet-initialized affinity hint - log "Setting ${smp_affinity} to ${affinity_cpumask} for device ${dev}" - echo "${affinity_cpumask}" > "${smp_affinity}" - fi - done -done |