diff options
Diffstat (limited to 'gcimagebundle')
32 files changed, 3826 insertions, 0 deletions
diff --git a/gcimagebundle/LICENSE b/gcimagebundle/LICENSE new file mode 100644 index 0000000..04cb0d7 --- /dev/null +++ b/gcimagebundle/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2013 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/gcimagebundle/MANIFEST.in b/gcimagebundle/MANIFEST.in new file mode 100644 index 0000000..6bbb29c --- /dev/null +++ b/gcimagebundle/MANIFEST.in @@ -0,0 +1,4 @@ +include *.md +include distribute_setup.py +include LICENSE +include VERSION diff --git a/gcimagebundle/README b/gcimagebundle/README new file mode 100644 index 0000000..13afc26 --- /dev/null +++ b/gcimagebundle/README @@ -0,0 +1,30 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Image bundling tool for root file system. + +Note: This tool is deprecated. Please see alternate instructions at +https://cloud.google.com/compute/docs/creating-custom-image#export_an_image_to_google_cloud_storage + +To build a root filesystem tar +$ sudo gcimagebundle -d /dev/sda -r / -o /tmp \ + --loglevel=DEBUG --log_file=/tmp/gcimagebundle.log + +This will output the image tar in the output directory +specified with -o option. + +Note that this is copied out file by file into the default google image. + +To run unittest: +$ sudo python setup.py test diff --git a/gcimagebundle/README.md b/gcimagebundle/README.md new file mode 100644 index 0000000..941a9d3 --- /dev/null +++ b/gcimagebundle/README.md @@ -0,0 +1,48 @@ +Image Bundle +============ + +_Image Bundle is deprecated and is provided here as is with no further support or maintenance. Please see alternate instructions for [Exporting an image to Google Cloud Storage](https://cloud.google.com/compute/docs/creating-custom-image#export_an_image_to_google_cloud_storage)._ + +Image Bundle is a python package that allows users to create an image from the current state of the running virtual machine. Image Bundle creates the image with the recommended packaging format and also allows you to run unit tests to verify that image bundle works properly on your operating system. See [Custom Images](https://cloud.google.com/compute/docs/creating-custom-image) for more information. + +### Installation + + $ sudo python setup.py install + +### Usage + +To build a root filesystem tar: + + $ sudo gcimagebundle -d /dev/sda -r / -o /tmp \ + --loglevel=DEBUG --log_file=/tmp/image_bundle.log + +This will output the image tar in the output directory specified with -o option. + +For details on all the parameters use... + + $ sudo gcimagebundle --help + +### Unit Tests + +Image Bundle includes unit tests that should be run if you make any changes. These tests perform mount operations so root access is required. + + $ sudo python setup.py test + +### Packaging + +Since Image Bundle uses setuptools it can be packaged into a DEB or RPM. + +Install the required dependencies: + + # For Debian based distributions + $ sudo apt-get install python-stdeb rpm + # For Red-Hat based distributions + $ sudo yum install rpmbuild + +DEB package: + + $ python setup.py --command-packages=stdeb.command bdist_deb + +RPM package: + + $ python setup.py bdist_rpm diff --git a/gcimagebundle/VERSION b/gcimagebundle/VERSION new file mode 100644 index 0000000..3a3cd8c --- /dev/null +++ b/gcimagebundle/VERSION @@ -0,0 +1 @@ +1.3.1 diff --git a/gcimagebundle/distribute_setup.py b/gcimagebundle/distribute_setup.py new file mode 100644 index 0000000..3553b21 --- /dev/null +++ b/gcimagebundle/distribute_setup.py @@ -0,0 +1,556 @@ +#!python +"""Bootstrap distribute installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from distribute_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" +import os +import shutil +import sys +import time +import fnmatch +import tempfile +import tarfile +import optparse + +from distutils import log + +try: + from site import USER_SITE +except ImportError: + USER_SITE = None + +try: + import subprocess + + def _python_cmd(*args): + args = (sys.executable,) + args + return subprocess.call(args) == 0 + +except ImportError: + # will be used for python 2.3 + def _python_cmd(*args): + args = (sys.executable,) + args + # quoting arguments if windows + if sys.platform == 'win32': + def quote(arg): + if ' ' in arg: + return '"%s"' % arg + return arg + args = [quote(arg) for arg in args] + return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 + +DEFAULT_VERSION = "0.6.49" +DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" +SETUPTOOLS_FAKED_VERSION = "0.6c11" + +SETUPTOOLS_PKG_INFO = """\ +Metadata-Version: 1.0 +Name: setuptools +Version: %s +Summary: xxxx +Home-page: xxx +Author: xxx +Author-email: xxx +License: xxx +Description: xxx +""" % SETUPTOOLS_FAKED_VERSION + + +def _install(tarball, install_args=()): + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # installing + log.warn('Installing Distribute') + if not _python_cmd('setup.py', 'install', *install_args): + log.warn('Something went wrong during the installation.') + log.warn('See the error message above.') + # exitcode will be 2 + return 2 + finally: + os.chdir(old_wd) + shutil.rmtree(tmpdir) + + +def _build_egg(egg, tarball, to_dir): + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # building an egg + log.warn('Building a Distribute egg in %s', to_dir) + _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + + finally: + os.chdir(old_wd) + shutil.rmtree(tmpdir) + # returning the result + log.warn(egg) + if not os.path.exists(egg): + raise IOError('Could not build the egg.') + + +def _do_download(version, download_base, to_dir, download_delay): + egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg' + % (version, sys.version_info[0], sys.version_info[1])) + if not os.path.exists(egg): + tarball = download_setuptools(version, download_base, + to_dir, download_delay) + _build_egg(egg, tarball, to_dir) + sys.path.insert(0, egg) + import setuptools + setuptools.bootstrap_install_from = egg + + +def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, download_delay=15, no_fake=True): + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + was_imported = 'pkg_resources' in sys.modules or \ + 'setuptools' in sys.modules + try: + try: + import pkg_resources + + # Setuptools 0.7b and later is a suitable (and preferable) + # substitute for any Distribute version. + try: + pkg_resources.require("setuptools>=0.7b") + return + except (pkg_resources.DistributionNotFound, + pkg_resources.VersionConflict): + pass + + if not hasattr(pkg_resources, '_distribute'): + if not no_fake: + _fake_setuptools() + raise ImportError + except ImportError: + return _do_download(version, download_base, to_dir, download_delay) + try: + pkg_resources.require("distribute>=" + version) + return + except pkg_resources.VersionConflict: + e = sys.exc_info()[1] + if was_imported: + sys.stderr.write( + "The required version of distribute (>=%s) is not available,\n" + "and can't be installed while this script is running. Please\n" + "install a more recent version first, using\n" + "'easy_install -U distribute'." + "\n\n(Currently using %r)\n" % (version, e.args[0])) + sys.exit(2) + else: + del pkg_resources, sys.modules['pkg_resources'] # reload ok + return _do_download(version, download_base, to_dir, + download_delay) + except pkg_resources.DistributionNotFound: + return _do_download(version, download_base, to_dir, + download_delay) + finally: + if not no_fake: + _create_fake_setuptools_pkg_info(to_dir) + + +def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, delay=15): + """Download distribute from a specified location and return its filename + + `version` should be a valid distribute version number that is available + as an egg for download under the `download_base` URL (which should end + with a '/'). `to_dir` is the directory where the egg will be downloaded. + `delay` is the number of seconds to pause before an actual download + attempt. + """ + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + try: + from urllib.request import urlopen + except ImportError: + from urllib2 import urlopen + tgz_name = "distribute-%s.tar.gz" % version + url = download_base + tgz_name + saveto = os.path.join(to_dir, tgz_name) + src = dst = None + if not os.path.exists(saveto): # Avoid repeated downloads + try: + log.warn("Downloading %s", url) + src = urlopen(url) + # Read/write all in one block, so we don't create a corrupt file + # if the download is interrupted. + data = src.read() + dst = open(saveto, "wb") + dst.write(data) + finally: + if src: + src.close() + if dst: + dst.close() + return os.path.realpath(saveto) + + +def _no_sandbox(function): + def __no_sandbox(*args, **kw): + try: + from setuptools.sandbox import DirectorySandbox + if not hasattr(DirectorySandbox, '_old'): + def violation(*args): + pass + DirectorySandbox._old = DirectorySandbox._violation + DirectorySandbox._violation = violation + patched = True + else: + patched = False + except ImportError: + patched = False + + try: + return function(*args, **kw) + finally: + if patched: + DirectorySandbox._violation = DirectorySandbox._old + del DirectorySandbox._old + + return __no_sandbox + + +def _patch_file(path, content): + """Will backup the file then patch it""" + f = open(path) + existing_content = f.read() + f.close() + if existing_content == content: + # already patched + log.warn('Already patched.') + return False + log.warn('Patching...') + _rename_path(path) + f = open(path, 'w') + try: + f.write(content) + finally: + f.close() + return True + +_patch_file = _no_sandbox(_patch_file) + + +def _same_content(path, content): + f = open(path) + existing_content = f.read() + f.close() + return existing_content == content + + +def _rename_path(path): + new_name = path + '.OLD.%s' % time.time() + log.warn('Renaming %s to %s', path, new_name) + os.rename(path, new_name) + return new_name + + +def _remove_flat_installation(placeholder): + if not os.path.isdir(placeholder): + log.warn('Unkown installation at %s', placeholder) + return False + found = False + for file in os.listdir(placeholder): + if fnmatch.fnmatch(file, 'setuptools*.egg-info'): + found = True + break + if not found: + log.warn('Could not locate setuptools*.egg-info') + return + + log.warn('Moving elements out of the way...') + pkg_info = os.path.join(placeholder, file) + if os.path.isdir(pkg_info): + patched = _patch_egg_dir(pkg_info) + else: + patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) + + if not patched: + log.warn('%s already patched.', pkg_info) + return False + # now let's move the files out of the way + for element in ('setuptools', 'pkg_resources.py', 'site.py'): + element = os.path.join(placeholder, element) + if os.path.exists(element): + _rename_path(element) + else: + log.warn('Could not find the %s element of the ' + 'Setuptools distribution', element) + return True + +_remove_flat_installation = _no_sandbox(_remove_flat_installation) + + +def _after_install(dist): + log.warn('After install bootstrap.') + placeholder = dist.get_command_obj('install').install_purelib + _create_fake_setuptools_pkg_info(placeholder) + + +def _create_fake_setuptools_pkg_info(placeholder): + if not placeholder or not os.path.exists(placeholder): + log.warn('Could not find the install location') + return + pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) + setuptools_file = 'setuptools-%s-py%s.egg-info' % \ + (SETUPTOOLS_FAKED_VERSION, pyver) + pkg_info = os.path.join(placeholder, setuptools_file) + if os.path.exists(pkg_info): + log.warn('%s already exists', pkg_info) + return + + log.warn('Creating %s', pkg_info) + try: + f = open(pkg_info, 'w') + except EnvironmentError: + log.warn("Don't have permissions to write %s, skipping", pkg_info) + return + try: + f.write(SETUPTOOLS_PKG_INFO) + finally: + f.close() + + pth_file = os.path.join(placeholder, 'setuptools.pth') + log.warn('Creating %s', pth_file) + f = open(pth_file, 'w') + try: + f.write(os.path.join(os.curdir, setuptools_file)) + finally: + f.close() + +_create_fake_setuptools_pkg_info = _no_sandbox( + _create_fake_setuptools_pkg_info +) + + +def _patch_egg_dir(path): + # let's check if it's already patched + pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') + if os.path.exists(pkg_info): + if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): + log.warn('%s already patched.', pkg_info) + return False + _rename_path(path) + os.mkdir(path) + os.mkdir(os.path.join(path, 'EGG-INFO')) + pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') + f = open(pkg_info, 'w') + try: + f.write(SETUPTOOLS_PKG_INFO) + finally: + f.close() + return True + +_patch_egg_dir = _no_sandbox(_patch_egg_dir) + + +def _before_install(): + log.warn('Before install bootstrap.') + _fake_setuptools() + + +def _under_prefix(location): + if 'install' not in sys.argv: + return True + args = sys.argv[sys.argv.index('install') + 1:] + for index, arg in enumerate(args): + for option in ('--root', '--prefix'): + if arg.startswith('%s=' % option): + top_dir = arg.split('root=')[-1] + return location.startswith(top_dir) + elif arg == option: + if len(args) > index: + top_dir = args[index + 1] + return location.startswith(top_dir) + if arg == '--user' and USER_SITE is not None: + return location.startswith(USER_SITE) + return True + + +def _fake_setuptools(): + log.warn('Scanning installed packages') + try: + import pkg_resources + except ImportError: + # we're cool + log.warn('Setuptools or Distribute does not seem to be installed.') + return + ws = pkg_resources.working_set + try: + setuptools_dist = ws.find( + pkg_resources.Requirement.parse('setuptools', replacement=False) + ) + except TypeError: + # old distribute API + setuptools_dist = ws.find( + pkg_resources.Requirement.parse('setuptools') + ) + + if setuptools_dist is None: + log.warn('No setuptools distribution found') + return + # detecting if it was already faked + setuptools_location = setuptools_dist.location + log.warn('Setuptools installation detected at %s', setuptools_location) + + # if --root or --preix was provided, and if + # setuptools is not located in them, we don't patch it + if not _under_prefix(setuptools_location): + log.warn('Not patching, --root or --prefix is installing Distribute' + ' in another location') + return + + # let's see if its an egg + if not setuptools_location.endswith('.egg'): + log.warn('Non-egg installation') + res = _remove_flat_installation(setuptools_location) + if not res: + return + else: + log.warn('Egg installation') + pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') + if (os.path.exists(pkg_info) and + _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): + log.warn('Already patched.') + return + log.warn('Patching...') + # let's create a fake egg replacing setuptools one + res = _patch_egg_dir(setuptools_location) + if not res: + return + log.warn('Patching complete.') + _relaunch() + + +def _relaunch(): + log.warn('Relaunching...') + # we have to relaunch the process + # pip marker to avoid a relaunch bug + _cmd1 = ['-c', 'install', '--single-version-externally-managed'] + _cmd2 = ['-c', 'install', '--record'] + if sys.argv[:3] == _cmd1 or sys.argv[:3] == _cmd2: + sys.argv[0] = 'setup.py' + args = [sys.executable] + sys.argv + sys.exit(subprocess.call(args)) + + +def _extractall(self, path=".", members=None): + """Extract all members from the archive to the current working + directory and set owner, modification time and permissions on + directories afterwards. `path' specifies a different directory + to extract to. `members' is optional and must be a subset of the + list returned by getmembers(). + """ + import copy + import operator + from tarfile import ExtractError + directories = [] + + if members is None: + members = self + + for tarinfo in members: + if tarinfo.isdir(): + # Extract directories with a safe mode. + directories.append(tarinfo) + tarinfo = copy.copy(tarinfo) + tarinfo.mode = 448 # decimal for oct 0700 + self.extract(tarinfo, path) + + # Reverse sort directories. + if sys.version_info < (2, 4): + def sorter(dir1, dir2): + return cmp(dir1.name, dir2.name) + directories.sort(sorter) + directories.reverse() + else: + directories.sort(key=operator.attrgetter('name'), reverse=True) + + # Set correct owner, mtime and filemode on directories. + for tarinfo in directories: + dirpath = os.path.join(path, tarinfo.name) + try: + self.chown(tarinfo, dirpath) + self.utime(tarinfo, dirpath) + self.chmod(tarinfo, dirpath) + except ExtractError: + e = sys.exc_info()[1] + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % e) + + +def _build_install_args(options): + """ + Build the arguments to 'python setup.py install' on the distribute package + """ + install_args = [] + if options.user_install: + if sys.version_info < (2, 6): + log.warn("--user requires Python 2.6 or later") + raise SystemExit(1) + install_args.append('--user') + return install_args + +def _parse_args(): + """ + Parse the command line for options + """ + parser = optparse.OptionParser() + parser.add_option( + '--user', dest='user_install', action='store_true', default=False, + help='install in user site package (requires Python 2.6 or later)') + parser.add_option( + '--download-base', dest='download_base', metavar="URL", + default=DEFAULT_URL, + help='alternative URL from where to download the distribute package') + options, args = parser.parse_args() + # positional arguments are ignored + return options + +def main(version=DEFAULT_VERSION): + """Install or upgrade setuptools and EasyInstall""" + options = _parse_args() + tarball = download_setuptools(download_base=options.download_base) + return _install(tarball, _build_install_args(options)) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/gcimagebundle/gcimagebundle b/gcimagebundle/gcimagebundle new file mode 100755 index 0000000..3ab7ec1 --- /dev/null +++ b/gcimagebundle/gcimagebundle @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Wrapper for gcimagebundle main.""" + +from gcimagebundlelib import imagebundle + + +def main(): + imagebundle.main() + + +if __name__ == '__main__': + main() diff --git a/gcimagebundle/gcimagebundlelib/__init__.py b/gcimagebundle/gcimagebundlelib/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/__init__.py diff --git a/gcimagebundle/gcimagebundlelib/block_disk.py b/gcimagebundle/gcimagebundlelib/block_disk.py new file mode 100644 index 0000000..a860b89 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/block_disk.py @@ -0,0 +1,389 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to create raw disk images. + +Stores a copy of directories/files in a file mounted as a partitioned blocked +device. +""" + + + +import hashlib +import logging +import os +import re +import tempfile + +from gcimagebundlelib import exclude_spec +from gcimagebundlelib import fs_copy +from gcimagebundlelib import utils + + +class RawDiskError(Exception): + """Error occured during raw disk creation.""" + + +class InvalidRawDiskError(Exception): + """Error when verification fails before copying.""" + + +class FsRawDisk(fs_copy.FsCopy): + """Creates a raw disk copy of OS image and bundles it into gzipped tar.""" + + def __init__(self, fs_size, fs_type): + """Constructor for FsRawDisk class. + + Args: + fs_size: Size of the raw disk. + """ + super(FsRawDisk, self).__init__() + self._fs_size = fs_size + self._fs_type = fs_type + + def _ResizeFile(self, file_path, file_size): + logging.debug('Resizing %s to %s', file_path, file_size) + with open(file_path, 'a') as disk_file: + disk_file.truncate(file_size) + + def _InitializeDiskFileFromDevice(self, file_path): + """Initializes disk file from the device specified in self._disk. + + It preserves whatever may be there on the device prior to the start of the + first partition. + + At the moment this method supports devices with a single partition only. + + Args: + file_path: The path where the disk file should be created. + + Returns: + A tuple with partition_start, uuid. partition_start is the location + where the first partition on the disk starts and uuid is the filesystem + UUID to use for the first partition. + + Raises: + RawDiskError: If there are more than one partition on the disk device. + """ + # Find the disk size + disk_size = utils.GetDiskSize(self._disk) + logging.debug('Size of disk is %s', disk_size) + # Make the disk file big enough to hold the disk + self._ResizeFile(file_path, disk_size) + # Find the location where the first partition starts + partition_start = utils.GetPartitionStart(self._disk, 1) + logging.debug('First partition starts at %s', partition_start) + # Copy all the bytes as is from the start of the disk to the start of + # first partition + utils.CopyBytes(self._disk, file_path, partition_start) + # Verify there is only 1 partition on the disk + with utils.LoadDiskImage(file_path) as devices: + # For now we only support disks with a single partition. + if len(devices) == 0: + raise RawDiskError( + 'Device %s should be a disk not a partition.' % self._disk) + elif len(devices) != 1: + raise RawDiskError( + 'Device %s has more than 1 partition. Only devices ' + 'with a single partition are supported.' % self._disk) + # Remove the first partition from the file we are creating. We will + # recreate a partition that will fit inside _fs_size later. + utils.RemovePartition(file_path, 1) + # Resize the disk.raw file down to self._fs_size + # We do this after removing the first partition to ensure that an + # existing partition doesn't fall outside the boundary of the disk device. + self._ResizeFile(file_path, self._fs_size) + # Get UUID of the first partition on the disk + # TODO(user): This is very hacky and relies on the disk path being + # similar to /dev/sda etc which is bad. Need to fix it. + uuid = utils.GetUUID(self._disk + '1') + return partition_start, uuid + + def Bundleup(self): + """Creates a raw disk copy of OS image and bundles it into gzipped tar. + + Returns: + A size of a generated raw disk and the SHA1 digest of the the tar archive. + + Raises: + RawDiskError: If number of partitions in a created image doesn't match + expected count. + """ + + # Create sparse file with specified size + disk_file_path = os.path.join(self._scratch_dir, 'disk.raw') + with open(disk_file_path, 'wb') as _: + pass + self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path)) + + logging.info('Initializing disk file') + partition_start = None + uuid = None + if self._disk: + # If a disk device has been provided then preserve whatever is there on + # the disk before the first partition in case there is an MBR present. + partition_start, uuid = self._InitializeDiskFileFromDevice(disk_file_path) + else: + # User didn't specify a disk device. Initialize a device with a simple + # partition table. + self._ResizeFile(disk_file_path, self._fs_size) + # User didn't specify a disk to copy. Create a new partition table + utils.MakePartitionTable(disk_file_path) + # Pass 1MB as start to avoid 'Warning: The resulting partition is not + # properly aligned for best performance.' from parted. + partition_start = 1024 * 1024 + + # Create a new partition starting at partition_start of size + # self._fs_size - partition_start + utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start, + self._fs_size - partition_start) + with utils.LoadDiskImage(disk_file_path) as devices: + # For now we only support disks with a single partition. + if len(devices) != 1: + raise RawDiskError(devices) + # List contents of /dev/mapper to help with debugging. Contents will + # be listed in debug log only + utils.RunCommand(['ls', '/dev/mapper']) + logging.info('Making filesystem') + uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid) + with utils.LoadDiskImage(disk_file_path) as devices: + if uuid is None: + raise Exception('Could not get uuid from MakeFileSystem') + mount_point = tempfile.mkdtemp(dir=self._scratch_dir) + with utils.MountFileSystem(devices[0], mount_point, self._fs_type): + logging.info('Copying contents') + self._CopySourceFiles(mount_point) + self._CopyPlatformSpecialFiles(mount_point) + self._ProcessOverwriteList(mount_point) + self._CleanupNetwork(mount_point) + self._UpdateFstab(mount_point, uuid) + + tar_entries = [] + + manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json') + manifest_created = self._manifest.CreateIfNeeded(manifest_file_path) + if manifest_created: + tar_entries.append(manifest_file_path) + + tar_entries.append(disk_file_path) + logging.info('Creating tar.gz archive') + utils.TarAndGzipFile(tar_entries, + self._output_tarfile) + for tar_entry in tar_entries: + os.remove(tar_entry) + + # TODO(user): It would be better to compute tar.gz file hash during + # archiving. + h = hashlib.sha1() + with open(self._output_tarfile, 'rb') as tar_file: + for chunk in iter(lambda: tar_file.read(8192), ''): + h.update(chunk) + return (self._fs_size, h.hexdigest()) + + def _CopySourceFiles(self, mount_point): + """Copies all source files/directories to a mounted raw disk. + + There are several cases which must be handled separately: + 1. src=dir1 and dest is empty. In this case we simply copy the content of + dir1 to mount_point. + 2. src=dir1 and dest=dir2. In this case dir1 is copied to mount_point + under a new name dir2, so its content would be copied under + mount_point/dir2. + 3. src=file1/dir1 and dest=file2/dir2 and is_recursive=False. file1/dir1 + is copied to mount_point/file2 or mount_point/dir2. + + Args: + mount_point: A path to a mounted raw disk. + """ + for (src, dest, is_recursive) in self._srcs: + # Generate a list of files/directories excluded from copying to raw disk. + # rsync expects them to be relative to src directory so we need to + # regenerate this list for every src separately. + with tempfile.NamedTemporaryFile(dir=self._scratch_dir) as rsync_file: + for spec in self._excludes: + rsync_file.write(spec.GetRsyncSpec(src)) + + # make sure that rsync utility sees all the content of rsync_file which + # otherwise can be buffered. + rsync_file.flush() + if is_recursive: + # if a directory ends with / rsync copies the content of a + # directory, otherwise it also copies the directory itself. + src = src.rstrip('/') + if not dest: + src += '/' + utils.Rsync(src, mount_point, rsync_file.name, + self._ignore_hard_links, recursive=True, xattrs=True) + if dest: + os.rename(os.path.join(mount_point, os.path.basename(src)), + os.path.join(mount_point, dest)) + else: + utils.Rsync(src, os.path.join(mount_point, dest), rsync_file.name, + self._ignore_hard_links, recursive=False, xattrs=True) + + def _CopyPlatformSpecialFiles(self, mount_point): + """Copies platform special files to a mounted raw disk. + + Args: + mount_point: A path to a mounted raw disk. + """ + if self._platform: + special_files = self._platform.GetPlatformSpecialFiles(self._scratch_dir) + for (src, dest) in special_files: + # Ensure we don't use extended attributes here, so that copying /selinux + # on Linux doesn't try and fail to preserve the SELinux context. That + # doesn't work and causes rsync to return a nonzero status code. + utils.Rsync(src, os.path.join(mount_point, dest), None, + self._ignore_hard_links, recursive=False, xattrs=False) + + def _ProcessOverwriteList(self, mount_point): + """Overwrites a set of files/directories requested by platform. + + Args: + mount_point: A path to a mounted raw disk. + """ + for file_name in self._overwrite_list: + file_path = os.path.join(mount_point, file_name) + if os.path.exists(file_path): + if os.path.isdir(file_path): + # TODO(user): platform.Overwrite is expected to overwrite the + # directory in place from what I can tell. In case of a file it will + # create a new file which must be copied to mounted raw disk. So there + # some inconsistency which would need to be addresses if and when we + # encounter a platform which would want to overwrite a directory. + self._platform.Overwrite(file_path, file_name, self._scratch_dir) + logging.info('rawdisk: modifying directory %s', file_path) + else: + new_file = self._platform.Overwrite(file_path, file_name, + self._scratch_dir) + logging.info('rawdisk: modifying %s from %s', file_path, new_file) + utils.Rsync(new_file, file_path, None, self._ignore_hard_links, + recursive=False, xattrs=True) + + + def _CleanupNetwork(self, mount_point): + """Remove any record of our current MAC address.""" + net_rules_path = os.path.join( + mount_point, + 'lib/udev/rules.d/75-persistent-net-generator.rules') + if os.path.exists(net_rules_path): + os.remove(net_rules_path) + + def _UpdateFstab(self, mount_point, uuid): + """Update /etc/fstab with the new root fs UUID.""" + fstab_path = os.path.join(mount_point, 'etc/fstab') + if not os.path.exists(fstab_path): + logging.warning('etc/fstab does not exist. Not updating fstab uuid') + return + + f = open(fstab_path, 'r') + lines = f.readlines() + f.close() + + def UpdateUUID(line): + """Replace the UUID on the entry for /.""" + g = re.match(r'UUID=\S+\s+/\s+(.*)', line) + if not g: + return line + return 'UUID=%s / %s\n' % (uuid, g.group(1)) + + logging.debug('Original /etc/fstab contents:\n%s', lines) + updated_lines = map(UpdateUUID, lines) + if lines == updated_lines: + logging.debug('No changes required to /etc/fstab') + return + logging.debug('Updated /etc/fstab contents:\n%s', updated_lines) + f = open(fstab_path, 'w') + f.write(''.join(updated_lines)) + f.close() + + +class RootFsRaw(FsRawDisk): + """Block disk copy of the root file system. + + Takes care of additional checks for a root file system. + """ + + def __init__( + self, fs_size, fs_type, skip_disk_space_check, statvfs = os.statvfs): + # statvfs parameter is for unit test to mock out os.statvfs call. + super(RootFsRaw, self).__init__(fs_size, fs_type) + self._skip_disk_space_check = skip_disk_space_check + self._statvfs = statvfs + + def _Verify(self): + super(RootFsRaw, self)._Verify() + # exactly one file system to bundle up + if len(self._srcs) != 1: + raise InvalidRawDiskError('Root filesystems must have exactly one src.') + # check that destination field is empty. + if self._srcs[0][1]: + raise InvalidRawDiskError('Root filesystems must be copied as /') + if (not self._skip_disk_space_check and + self._srcs[0][0] == '/'): + self._VerifyDiskSpace() + + def _VerifyDiskSpace(self): + """Verify that there is enough free disk space to generate the image file""" + # We use a very quick and simplistic check, + # DiskSpaceNeeded = disk.raw + image.tar.gz + LogFile + # disk.raw = PartitionTable + AllFilesCopied + # AllFilesCopied = RootDiskSize - RootDiskFreeSize - ExcludedFiles + # We ignore LogFile, PartitionTable, and ExcludedFiles. + # Some empirical experience showed that the compression ratio of the + # tar.gz file is about 1/3. To be conservative, we assume image.tar.gz is + # about 40% of disk.raw file. + # As a result, DiskSpaceNeeded=1.4*(RootDiskSize - RootDiskFreeSize) + # TODO(user): Make this check more accurate because ignoring ExcludedFiles + # can result in significant overestimation of disk + # space needed if the user has large disk space used in /tmp, for example. + root_fs = self._statvfs(self._srcs[0][0]) + disk_space_needed = long(1.4 * root_fs.f_bsize * (root_fs.f_blocks - + root_fs.f_bfree)) + logging.info(("Root disk on %s: f_bsize=%d f_blocks=%d f_bfree=%d. " + "Estimated space needed is %d (may be overestimated)."), + self._srcs[0][0], + root_fs.f_bsize, + root_fs.f_blocks, + root_fs.f_bfree, + disk_space_needed) + + # self._scratch_dir is where we will put the disk.raw and *.tar.gz file. + scratch_fs = self._statvfs(self._scratch_dir) + free_space = scratch_fs.f_bsize * scratch_fs.f_bfree + logging.info("Free disk space for %s is %d bytes.", + self._scratch_dir, + free_space) + + if disk_space_needed > free_space: + errorMessage = ("The operation may require up to %d bytes of disk space. " + "However, the free disk space for %s is %d bytes. Please consider " + "freeing more disk space. Note that the disk space required may " + "be overestimated because it does not exclude temporary files that " + "will not be copied. You may use --skip_disk_space_check to disable " + "this check.") % (disk_space_needed, self._scratch_dir, free_space) + raise InvalidRawDiskError(errorMessage) + if disk_space_needed > self._fs_size: + errorMessage = ("The root disk files to be copied may require up to %d " + "bytes. However, the limit on the image disk file is %d bytes. " + "Please consider deleting unused files from root disk, " + "or increasing the image disk file limit with --fssize option. " + "Note that the disk space required may " + "be overestimated because it does not exclude temporary files that " + "will not be copied. You may use --skip_disk_space_check to disable " + "this check.") % (disk_space_needed, self._fs_size) + raise InvalidRawDiskError(errorMessage) + + + diff --git a/gcimagebundle/gcimagebundlelib/centos.py b/gcimagebundle/gcimagebundlelib/centos.py new file mode 100644 index 0000000..1a082de --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/centos.py @@ -0,0 +1,66 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Centos specific platform info.""" + + + +import os +import platform +import re + +from gcimagebundlelib import linux + + +class Centos(linux.LinuxPlatform): + """Centos specific information.""" + + @staticmethod + def IsThisPlatform(root='/'): + release_file = root + '/etc/redhat-release' + if os.path.exists(release_file): + (_, _, flavor, _) = Centos.ParseRedhatRelease(release_file) + if flavor and flavor.lower() == 'centos': + return True + return False + + @staticmethod + def ParseRedhatRelease(release_file='/etc/redhat-release'): + """Parses the /etc/redhat-release file.""" + f = open(release_file) + lines = f.readlines() + f.close() + if not lines: + return (None, None, None, None) + line0 = lines[0] + # Matches both CentOS 6 and CentOS 7 formats. + # CentOS 6: CentOS release 6.5 (Final) + # CentOS 7: CentOS Linux release 7.0.1406 (Core) + g = re.match(r'(\S+)( Linux)? release (\d+(\.\d+)+) \(([^)]*)\)', line0) + if not g: + return (None, None, None, None) + (osname, version, label) = (g.group(1), g.group(3), g.group(5)) + return (osname, label, osname, version) + + def __init__(self): + super(Centos, self).__init__() + (self.distribution_codename, _, self.distribution, + self.distribution_version) = Centos.ParseRedhatRelease() + + def GetPreferredFilesystemType(self): + (_,version,_) = platform.linux_distribution() + if version.startswith('7'): + return 'xfs' + return 'ext4' diff --git a/gcimagebundle/gcimagebundlelib/debian.py b/gcimagebundle/gcimagebundlelib/debian.py new file mode 100644 index 0000000..957e3a7 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/debian.py @@ -0,0 +1,36 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Debian Linux specific platform info.""" + + + +import platform + +from gcimagebundlelib import linux + + +class Debian(linux.LinuxPlatform): + """Debian Linux specific information.""" + + @staticmethod + def IsThisPlatform(root='/'): + (distribution, _, _) = platform.linux_distribution() + if distribution and distribution.lower() == 'debian': + return True + return False + + def __init__(self): + super(Debian, self).__init__() diff --git a/gcimagebundle/gcimagebundlelib/exclude_spec.py b/gcimagebundle/gcimagebundlelib/exclude_spec.py new file mode 100644 index 0000000..b5bc237 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/exclude_spec.py @@ -0,0 +1,82 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Exclude file specification.""" + + + +import logging +import os + + +class ExcludeSpec(object): + """Specifies how exclusion of a path should be handled.""" + + def __init__(self, path, preserve_file=False, preserve_dir=False, + preserve_subdir=False): + self.path = path + self.preserve_dir = preserve_dir + self.preserve_file = False + self.preserve_subdir = False + # Preserve files and subdirs only if dir is preserved. + if preserve_file and preserve_dir: + self.preserve_file = True + if preserve_subdir and preserve_dir: + self.preserve_subdir = True + + def ShouldExclude(self, filename): + prefix = os.path.commonprefix([filename, self.path]) + if prefix == self.path: + if ((self.preserve_dir and filename == self.path) or + (self.preserve_subdir and os.path.isdir(filename)) or + (self.preserve_file and os.path.isfile(filename))): + logging.warning('preserving %s', filename) + return False + return True + return False + + def GetSpec(self): + return '(%s, %d:%d:%d)' % (self.path, self.preserve_file, self.preserve_dir, + self.preserve_subdir) + + def GetRsyncSpec(self, src): + """Returns exclude spec in a format required by rsync. + + Args: + src: source directory path passed to rsync. rsync expects exclude-spec to + be relative to src directory. + + Returns: + A string of exclude filters in rsync exclude-from file format. + """ + spec = '' + prefix = os.path.commonprefix([src, self.path]) + if prefix == src: + relative_path = os.path.join('/', self.path[len(prefix):]) + if self.preserve_dir: + spec += '+ %s\n' % relative_path + if self.preserve_file or self.preserve_subdir: + if os.path.isdir(self.path): + for f in os.listdir(self.path): + file_path = os.path.join(self.path, f) + relative_file_path = os.path.join(relative_path, f) + if self.preserve_file and os.path.isfile(file_path): + spec += '+ %s\n' % relative_file_path + if self.preserve_subdir and os.path.isdir(file_path): + spec += '+ %s\n' % relative_file_path + else: + spec += '- %s\n' % relative_path + spec += '- %s\n' % os.path.join(relative_path, '**') + return spec diff --git a/gcimagebundle/gcimagebundlelib/fedora.py b/gcimagebundle/gcimagebundlelib/fedora.py new file mode 100644 index 0000000..21d098b --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/fedora.py @@ -0,0 +1,56 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Fedora specific platform info.""" + + + +import os +import re + +from gcimagebundlelib import linux + + +class Fedora(linux.LinuxPlatform): + """Fedora specific information.""" + + @staticmethod + def IsThisPlatform(root='/'): + release_file = root + '/etc/redhat-release' + if os.path.exists(release_file): + (_, _, flavor, _) = Fedora.ParseRedhatRelease(release_file) + if flavor and flavor.lower() == 'fedora': + return True + return False + + @staticmethod + def ParseRedhatRelease(release_file='/etc/redhat-release'): + """Parses the /etc/redhat-release file.""" + f = open(release_file) + lines = f.readlines() + f.close() + if not lines: + return (None, None, None, None) + line0 = lines[0] + g = re.match(r'(\S+) release (\d+) \(([^)]*)\)', line0) + if not g: + return (None, None, None, None) + (osname, version, label) = (g.group(1), g.group(2), g.group(3)) + return (osname, label, osname, version) + + def __init__(self): + super(Fedora, self).__init__() + (self.distribution_codename, _, self.distribution, + self.distribution_version) = Fedora.ParseRedhatRelease() diff --git a/gcimagebundle/gcimagebundlelib/fs_copy.py b/gcimagebundle/gcimagebundlelib/fs_copy.py new file mode 100644 index 0000000..e9adc91 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/fs_copy.py @@ -0,0 +1,180 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Creates a copy of specified directories\files.""" + + + +import logging +import os +import re + +from gcimagebundlelib import manifest +from gcimagebundlelib import utils + + +class FsCopyError(Exception): + """Error occured in fs copy operation.""" + + +class InvalidFsCopyError(Exception): + """Error when verification fails before fs copying.""" + + +class FsCopy(object): + """Specifies which files/directories must be copied.""" + + def __init__(self): + # Populate the required parameters with None so we can verify. + self._output_tarfile = None + self._srcs = [] + self._excludes = [] + self._key = None + self._recursive = True + self._fs_size = 0 + self._ignore_hard_links = False + self._platform = None + self._overwrite_list = [] + self._scratch_dir = '/tmp' + self._disk = None + self._manifest = manifest.ImageManifest(is_gce_instance=utils.IsRunningOnGCE()) + + def SetTarfile(self, tar_file): + """Sets tar file which will contain file system copy. + + Args: + tar_file: path to a tar file. + """ + self._output_tarfile = tar_file + + def AddDisk(self, disk): + """Adds the disk which should be bundled. + + Args: + disk: The block disk that needs to be bundled. + """ + self._disk = disk + + def AddSource(self, src, arcname='', recursive=True): + """Adds a source to be copied to the tar file. + + Args: + src: path to directory/file to be copied. + arcname: name of src in the tar archive. If arcname is empty, then instead + of copying src itself only its content is copied. + recursive: specifies if src directory should be copied recursively. + + Raises: + ValueError: If src path doesn't exist. + """ + if not os.path.exists(src): + raise ValueError('invalid path') + # Note that there is a fundamental asymmetry here as + # abspath('/') => '/' while abspath('/usr/') => '/usr'. + # This creates some subtleties elsewhere in the code. + self._srcs.append((os.path.abspath(src), arcname, recursive)) + + def AppendExcludes(self, excludes): + """Adds a file/directory to be excluded from file copy. + + Args: + excludes: A list of ExcludeSpec objects. + """ + self._excludes.extend(excludes) + + def SetKey(self, key): + """Sets a key to use to sign the archive digest. + + Args: + key: key to use to sign the archive digest. + """ + # The key is ignored for now. + # TODO(user): sign the digest with the key + self._key = key + + def SetPlatform(self, platform): + """Sets the OS platform which is used to create an image. + + Args: + platform: OS platform specific settings. + """ + self._platform = platform + logging.warning('overwrite list = %s', + ' '.join(platform.GetOverwriteList())) + self._overwrite_list = [re.sub('^/', '', x) + for x in platform.GetOverwriteList()] + + def _SetManifest(self, image_manifest): + """For test only, allows to set a test manifest object.""" + self._manifest = image_manifest + + def SetScratchDirectory(self, directory): + """Sets a directory used for storing intermediate results. + + Args: + directory: scratch directory path. + """ + self._scratch_dir = directory + + def IgnoreHardLinks(self): + """Requests that hard links should not be copied as hard links.""" + + # TODO(user): I don't see a reason for this option to exist. Currently + # there is a difference in how this option is interpreted between FsTarball + # and FsRawDisk. FsTarball only copies one hard link to an inode and ignores + # the rest of them. FsRawDisk copies the content of a file that hard link is + # pointing to instead of recreating a hard link. Either option seems useless + # for creating a copy of a file system. + self._ignore_hard_links = True + + def Verify(self): + """Verify if we have all the components to build a tar.""" + self._Verify() + + def Bundleup(self): + """Creates the tar image based on set parameters. + + Returns: + the SHA1 digest of the the tar archive. + """ + return (0, None) + + def _Verify(self): + """Verifies the tar attributes. Raises InvalidTarballError. + + Raises: + InvalidFsCopyError: If not all required parameters are set. + FsCopyError: If source file does not exist. + """ + if not self._output_tarfile or not self._srcs or not self._key: + raise InvalidFsCopyError('Incomplete copy spec') + for (src, _, _) in self._srcs: + if not os.path.exists(src): + raise FsCopyError('%s does not exists' % src) + + def _ShouldExclude(self, filename): + """"Checks if a file/directory are excluded from a copy. + + Args: + filename: a file/directory path. + + Returns: + True if a file/directory shouldn't be copied, False otherwise. + """ + for spec in self._excludes: + if spec.ShouldExclude(filename): + logging.info('tarfile: Excluded %s', filename) + return True + return False diff --git a/gcimagebundle/gcimagebundlelib/gcel.py b/gcimagebundle/gcimagebundlelib/gcel.py new file mode 100644 index 0000000..2622cf7 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/gcel.py @@ -0,0 +1,57 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""GCE Linux specific platform info.""" + + + +import csv +import os + +from gcimagebundlelib import linux + + +class Gcel(linux.LinuxPlatform): + """GCE Linux specific information.""" + + @staticmethod + def IsThisPlatform(root='/'): + release_file = root + '/etc/lsb-release' + if os.path.exists(release_file): + (flavor, _, _, _) = Gcel.ParseLsbRelease(release_file) + if flavor and flavor.lower() == 'gcel': + return True + return False + + @staticmethod + def ParseLsbRelease(release_file='/etc/lsb-release'): + """Parses the /etc/lsb-releases file. + + Returns: + A 4-tuple containing id, release, codename, and description + """ + release_info = {} + for line in csv.reader(open(release_file), delimiter='='): + if len(line) > 1: + release_info[line[0]] = line[1] + return (release_info.get('DISTRIB_ID', None), + release_info.get('DISTRIB_RELEASE', None), + release_info.get('DISTRIB_CODENAME', None), + release_info.get('DISTRIB_DESCRIPTION', None)) + + def __init__(self): + super(Gcel, self).__init__() + (self.distribution, self.distribution_version, + self.distribution_codename, _) = Gcel.ParseLsbRelease() diff --git a/gcimagebundle/gcimagebundlelib/imagebundle.py b/gcimagebundle/gcimagebundlelib/imagebundle.py new file mode 100755 index 0000000..f275c3c --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/imagebundle.py @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tool to bundle root filesystem to a tarball. + +Creates a tar bundle and a Manifest, which can be uploaded to image store. +""" + + + +import logging +from optparse import OptionParser +import os +import shutil +import subprocess +import tempfile +import time + +from gcimagebundlelib import block_disk +from gcimagebundlelib import exclude_spec +from gcimagebundlelib import platform_factory +from gcimagebundlelib import utils + +def SetupArgsParser(): + """Sets up the command line flags.""" + parser = OptionParser() + parser.add_option('-d', '--disk', dest='disk', + default='/dev/sda', + help='Disk to bundle.') + parser.add_option('-r', '--root', dest='root_directory', + default='/', metavar='ROOT', + help='Root of the file system to bundle.' + ' Recursively bundles all sub directories.') + parser.add_option('-e', '--excludes', dest='excludes', + help='Comma separated list of sub directories to exclude.' + ' The defaults are platform specific.') + parser.add_option('-o', '--output_directory', dest='output_directory', + default='/tmp/', metavar='DIR', + help='Output directory for image.') + parser.add_option('--output_file_name', dest='output_file_name', + default=None, metavar='FILENAME', + help=('Output filename for the image. Default is a digest' + ' of the image bytes.')) + parser.add_option('--include_mounts', dest='include_mounts', + help='Don\'t ignore mounted filesystems under ROOT.', + action='store_true', default=False) + parser.add_option('-v', '--version', + action='store_true', dest='display_version', default=False, + help='Print the tool version.') + parser.add_option('--loglevel', dest='log_level', + help='Debug logging level.', default='INFO', + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR' 'CRITICAL']) + parser.add_option('--log_file', dest='log_file', + help='Output file for log messages.') + parser.add_option('-k', '--key', dest='key', default='nebula', + help='Public key used for signing the image.') + parser.add_option('--nocleanup', dest='cleanup', + action='store_false', default=True, + help=' Do not clean up temporary and log files.') + #TODO(user): Get dehumanize. + parser.add_option('--fssize', dest='fs_size', default=10*1024*1024*1024, + type='int', help='File system size in bytes') + parser.add_option('-b', '--bucket', dest='bucket', + help='Destination storage bucket') + parser.add_option('-f', '--filesystem', dest='file_system', + default=None, + help='File system type for the image.') + parser.add_option('--skip_disk_space_check', dest='skip_disk_space_check', + default=False, action='store_true', + help='Skip the disk space requirement check.') + + return parser + + +def VerifyArgs(parser, options): + """Verifies that commandline flags are consistent.""" + if not options.output_directory: + parser.error('output bundle directory must be specified.') + if not os.path.exists(options.output_directory): + parser.error('output bundle directory does not exist.') + + # TODO(user): add more verification as needed + +def EnsureSuperUser(): + """Ensures that current user has super user privileges.""" + if os.getuid() != 0: + logging.warning('Tool must be run as root.') + exit(-1) + + +def GetLogLevel(options): + """Log Level string to logging.LogLevel mapping.""" + level = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL + } + if options.log_level in level: + return level[options.log_level] + print 'Invalid logging level. defaulting to INFO.' + return logging.INFO + + +def SetupLogging(options, log_dir='/tmp'): + """Set up logging. + + All messages above INFO level are also logged to console. + + Args: + options: collection of command line options. + log_dir: directory used to generate log files. + """ + if options.log_file: + logfile = options.log_file + else: + logfile = tempfile.mktemp(dir=log_dir, prefix='bundle_log_') + print 'Starting logging in %s' % logfile + logging.basicConfig(filename=logfile, + level=GetLogLevel(options), + format='%(asctime)s %(levelname)s:%(name)s:%(message)s') + # Use GMT timestamp in logging. + logging.Formatter.converter=time.gmtime + console = logging.StreamHandler() + console.setLevel(GetLogLevel(options)) + logging.getLogger().addHandler(console) + + +def PrintVersionInfo(): + #TODO: Should read from the VERSION file instead. + print 'version 1.3.1' + + +def GetTargetFilesystem(options, guest_platform): + if options.file_system: + return options.file_system + else: + return guest_platform.GetPreferredFilesystemType() + + +def main(): + parser = SetupArgsParser() + (options, _) = parser.parse_args() + if options.display_version: + PrintVersionInfo() + return 0 + EnsureSuperUser() + VerifyArgs(parser, options) + + scratch_dir = tempfile.mkdtemp(dir=options.output_directory) + SetupLogging(options, scratch_dir) + logging.warn('============================================================\n' + 'Warning: gcimagebundle is deprecated. See\n' + 'https://cloud.google.com/compute/docs/creating-custom-image' + '#export_an_image_to_google_cloud_storage\n' + 'for updated instructions.\n' + '============================================================') + try: + guest_platform = platform_factory.PlatformFactory( + options.root_directory).GetPlatform() + except platform_factory.UnknownPlatformException: + logging.critical('Platform is not supported.' + ' Platform rules can be added to platform_factory.py.') + return -1 + + temp_file_name = tempfile.mktemp(dir=scratch_dir, suffix='.tar.gz') + + file_system = GetTargetFilesystem(options, guest_platform) + logging.info('File System: %s', file_system) + logging.info('Disk Size: %s bytes', options.fs_size) + bundle = block_disk.RootFsRaw( + options.fs_size, file_system, options.skip_disk_space_check) + bundle.SetTarfile(temp_file_name) + if options.disk: + readlink_command = ['readlink', '-f', options.disk] + final_path = utils.RunCommand(readlink_command).strip() + logging.info('Resolved %s to %s', options.disk, final_path) + bundle.AddDisk(final_path) + # TODO(user): Find the location where the first partition of the disk + # is mounted and add it as the source instead of relying on the source + # param flag + bundle.AddSource(options.root_directory) + bundle.SetKey(options.key) + bundle.SetScratchDirectory(scratch_dir) + + # Merge platform specific exclude list, mounts points + # and user specified excludes + excludes = guest_platform.GetExcludeList() + if options.excludes: + excludes.extend([exclude_spec.ExcludeSpec(x) for x in + options.excludes.split(',')]) + logging.info('exclude list: %s', ' '.join([x.GetSpec() for x in excludes])) + bundle.AppendExcludes(excludes) + if not options.include_mounts: + mount_points = utils.GetMounts(options.root_directory) + logging.info('ignoring mounts %s', ' '.join(mount_points)) + bundle.AppendExcludes([exclude_spec.ExcludeSpec(x, preserve_dir=True) for x + in utils.GetMounts(options.root_directory)]) + bundle.SetPlatform(guest_platform) + + # Verify that bundle attributes are correct and create tar bundle. + bundle.Verify() + (fs_size, digest) = bundle.Bundleup() + if not digest: + logging.critical('Could not get digest for the bundle.' + ' The bundle may not be created correctly') + return -1 + if fs_size > options.fs_size: + logging.critical('Size of tar %d exceeds the file system size %d.', fs_size, + options.fs_size) + return -1 + + if options.output_file_name: + output_file = os.path.join( + options.output_directory, options.output_file_name) + else: + output_file = os.path.join( + options.output_directory, '%s.image.tar.gz' % digest) + + os.rename(temp_file_name, output_file) + logging.info('Created tar.gz file at %s' % output_file) + + if options.bucket: + bucket = options.bucket + if bucket.startswith('gs://'): + output_bucket = '%s/%s' % ( + bucket, os.path.basename(output_file)) + else: + output_bucket = 'gs://%s/%s' % ( + bucket, os.path.basename(output_file)) + + # /usr/local/bin not in redhat root PATH by default + if '/usr/local/bin' not in os.environ['PATH']: + os.environ['PATH'] += ':/usr/local/bin' + + # TODO: Consider using boto library directly. + cmd = ['gsutil', 'cp', output_file, output_bucket] + retcode = subprocess.call(cmd) + if retcode != 0: + logging.critical('Failed to copy image to bucket. ' + 'gsutil returned %d. To retry, run the command: %s', + retcode, ' '.join(cmd)) + + return -1 + logging.info('Uploaded image to %s', output_bucket) + + # If we've uploaded, then we can remove the local file. + os.remove(output_file) + + if options.cleanup: + shutil.rmtree(scratch_dir) diff --git a/gcimagebundle/gcimagebundlelib/linux.py b/gcimagebundle/gcimagebundlelib/linux.py new file mode 100644 index 0000000..ff8c1d4 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/linux.py @@ -0,0 +1,135 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Base class for Linux platform specific information.""" + + + +import os +import platform +import stat + +from gcimagebundlelib import exclude_spec +from gcimagebundlelib import os_platform + + +class LinuxPlatform(os_platform.Platform): + """Base class for all Linux flavors.""" + EXCLUDE_LIST = [ + exclude_spec.ExcludeSpec('/etc/ssh/.host_key_regenerated'), + exclude_spec.ExcludeSpec('/dev', preserve_dir=True), + exclude_spec.ExcludeSpec('/proc', preserve_dir=True), + exclude_spec.ExcludeSpec('/run', + preserve_dir=True, preserve_subdir=True), + exclude_spec.ExcludeSpec('/selinux'), + exclude_spec.ExcludeSpec('/tmp', preserve_dir=True), + exclude_spec.ExcludeSpec('/sys', preserve_dir=True), + exclude_spec.ExcludeSpec('/var/lib/google/per-instance', + preserve_dir=True), + exclude_spec.ExcludeSpec('/var/lock', + preserve_dir=True, preserve_subdir=True), + exclude_spec.ExcludeSpec('/var/log', + preserve_dir=True, preserve_subdir=True), + exclude_spec.ExcludeSpec('/var/run', + preserve_dir=True, preserve_subdir=True)] + + def __init__(self): + """Populate the uname -a information.""" + super(LinuxPlatform, self).__init__() + (self.name, self.hostname, self.release, self.version, self.architecture, + self.processor) = platform.uname() + (self.distribution, self.distribution_version, + self.distribution_codename) = platform.dist() + + def GetPlatformDetails(self): + return ' '.join([self.name, self.hostname, self.release, self.version, + self.architecture, self.processor, self.distribution, + self.distribution_version, self.distribution_codename]) + + def GetName(self): + return self.GetOs() + + def GetProcessor(self): + return platform.processor() + + def GetArchitecture(self): + if self.architecture: + return self.architecture + return '' + + def GetOs(self): + if self.distribution: + if self.distribution_codename: + return '%s (%s)' % (self.distribution, self.distribution_codename) + else: + return self.distribution + if self.name: + return self.name + return 'Linux' + + def IsLinux(self): + return True + + # Linux specific methods + def GetKernelVersion(self): + return self.release + + # distribution specific methods + # if platforms module does not do a good job override these. + def GetDistribution(self): + return self.distribution + + def GetDistributionCodeName(self): + return self.distribution_codename + + def GetDistributionVersion(self): + return self.distribution_version + + def GetPlatformSpecialFiles(self, tmpdir='/tmp'): + """Creates any platform specific special files.""" + retval = [] + console_dev = os.makedev(5, 1) + os.mknod(tmpdir + 'console', stat.S_IFCHR | + stat.S_IRUSR | stat.S_IWUSR, console_dev) + retval.append((tmpdir + 'console', 'dev/console')) + null_dev = os.makedev(1, 3) + os.mknod(tmpdir + 'null', stat.S_IFCHR | + stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | + stat.S_IROTH | stat.S_IWOTH, null_dev) + retval.append((tmpdir + 'null', 'dev/null')) + tty_dev = os.makedev(5, 0) + os.mknod(tmpdir + 'tty', stat.S_IFCHR | + stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | + stat.S_IROTH | stat.S_IWOTH, tty_dev) + retval.append((tmpdir + 'tty', 'dev/tty')) + zero_dev = os.makedev(1, 5) + os.mknod(tmpdir + 'zero', stat.S_IFCHR | + stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | + stat.S_IROTH | stat.S_IWOTH, zero_dev) + retval.append((tmpdir + 'zero', 'dev/zero')) + # /selinux is deprecated in favor of /sys/fs/selinux, but preserve it on + # those OSes where it's present. + if os.path.isdir('/selinux'): + os.mkdir(tmpdir + 'selinux', 0755) + retval.append((tmpdir + 'selinux', 'selinux')) + return retval + + def Overwrite(self, filename, arcname, tmpdir='/tmp'): + """Overwrites specified file if needed for the Linux platform.""" + pass + + def GetPreferredFilesystemType(self): + """Return the optimal filesystem supported for the platform.""" + return 'ext4' diff --git a/gcimagebundle/gcimagebundlelib/manifest.py b/gcimagebundle/gcimagebundlelib/manifest.py new file mode 100755 index 0000000..2e83d9e --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/manifest.py @@ -0,0 +1,79 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Image manifest.""" + + +import json +from gcimagebundlelib import utils + + +class ImageManifest(object): + """Retrieves metadata from the instance and stores it in manifest.json. + + The image manifest is a JSON file that is bundled along side the disk. + + Included Metadata + - Licenses + """ + + def __init__(self, http=utils.Http(), is_gce_instance=True): + self._http = http + self._licenses = [] + self._is_gce_instance = is_gce_instance + + def CreateIfNeeded(self, file_path): + """Creates the manifest file to the specified path if it's needed. + + Args: + file_path: Location of where the manifest should be written to. + + Returns: + True Manifest was written to file_path. + False Manifest was not created. + """ + if self._is_gce_instance: + self._LoadLicenses() + if self._IsManifestNeeded(): + with open(file_path, 'w') as manifest_file: + self._WriteToFile(manifest_file) + return True + return False + + def _LoadLicenses(self): + """Loads the licenses from the metadata server if they exist.""" + response = self._http.GetMetadata('instance/', recursive=True) + instance_metadata = json.loads(response) + if 'licenses' in instance_metadata: + for license_obj in instance_metadata['licenses']: + self._licenses.append(license_obj['id']) + + def _ToJson(self): + """Formats the image metadata as a JSON object.""" + return json.dumps( + { + 'licenses': self._licenses + }) + + def _IsManifestNeeded(self): + """Determines if a manifest should be bundled with the disk.""" + if self._licenses: + return len(self._licenses) + return False + + def _WriteToFile(self, file_obj): + """Writes the manifest data to the file handle.""" + manifest_json = self._ToJson() + file_obj.write(manifest_json) diff --git a/gcimagebundle/gcimagebundlelib/opensuse.py b/gcimagebundle/gcimagebundlelib/opensuse.py new file mode 100644 index 0000000..9f709ff --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/opensuse.py @@ -0,0 +1,29 @@ +# Copyright 2013 SUSE LLC All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""openSUSE platform info.""" + +from gcimagebundlelib import suse + +class OpenSUSE(suse.SUSE): + """openSUSE platform info.""" + + @staticmethod + def IsThisPlatform(root='/'): + return 'openSUSE' == suse.SUSE().distribution + + def __init__(self): + super(OpenSUSE, self).__init__() + diff --git a/gcimagebundle/gcimagebundlelib/os_platform.py b/gcimagebundle/gcimagebundlelib/os_platform.py new file mode 100644 index 0000000..65e6e7c --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/os_platform.py @@ -0,0 +1,70 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Base class for platform specific information.""" + + +class Platform(object): + """Base class for platform information.""" + EXCLUDE_LIST = [] + OVERWRITE_LIST = [] + + @staticmethod + def IsThisPlatform(root='/'): + return False + + def __init__(self): + pass + + def GetName(self): + """Generic name for the platform.""" + return 'Unknown' + + def GetProcessor(self): + """Real processor.""" + return '' + + def GetArchitecture(self): + """Returns machine architecture.""" + return '' + + def GetExcludeList(self): + """Returns the default exclude list of the platform.""" + return self.__class__.EXCLUDE_LIST + + def GetOs(self): + """Returns the name of OS.""" + return 'Unknown' + + def IsLinux(self): + return False + + def IsWindows(self): + return False + + def IsUnix(self): + return False + + def GetOverwriteList(self): + """Returns list of platform specific files to overwrite.""" + return self.__class__.OVERWRITE_LIST + + def Overwrite(self, file_path, file_name, scratch_dir): + """Called for each file in the OverwriteList.""" + return file_name + + def GetPlatformSpecialFiles(self, tmpdir): + """returns a list of platform special files that should be created.""" + return [] diff --git a/gcimagebundle/gcimagebundlelib/platform_factory.py b/gcimagebundle/gcimagebundlelib/platform_factory.py new file mode 100644 index 0000000..da63f0e --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/platform_factory.py @@ -0,0 +1,60 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Factory that guesses the correct platform and creates it.""" + +import logging + +from gcimagebundlelib import centos +from gcimagebundlelib import fedora +from gcimagebundlelib import debian +from gcimagebundlelib import gcel +from gcimagebundlelib import opensuse +from gcimagebundlelib import rhel +from gcimagebundlelib import sle +from gcimagebundlelib import ubuntu + + +class UnknownPlatformException(Exception): + """The platform could not be correctly determined.""" + + +class PlatformFactory(object): + """Guess the platform and create it.""" + + def __init__(self, root='/'): + self.__root = root + self.__registry = {} + self.__platform_registry = {} + self.Register('Centos', centos.Centos) + self.Register('Fedora', fedora.Fedora) + self.Register('Debian', debian.Debian) + self.Register('GCEL', gcel.Gcel) + self.Register('openSUSE', opensuse.OpenSUSE) + self.Register('Red Hat Enterprise Linux', rhel.RHEL) + self.Register('SUSE Linux Enterprise', sle.SLE) + self.Register('Ubuntu', ubuntu.Ubuntu) + + def Register(self, name, klass): + self.__registry[name] = klass + + def GetPlatform(self): + for name in self.__registry: + if self.__registry[name].IsThisPlatform(self.__root): + logging.info('found platform %s', name) + return self.__registry[name]() + else: + logging.debug('skipping platform %s %s ', name, self.__registry[name]) + raise UnknownPlatformException('Could not determine host platform.') diff --git a/gcimagebundle/gcimagebundlelib/rhel.py b/gcimagebundle/gcimagebundlelib/rhel.py new file mode 100644 index 0000000..9ebf1ef --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/rhel.py @@ -0,0 +1,42 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Red Hat Enterprise Linux Linux specific platform info.""" + + + +import platform + +from gcimagebundlelib import linux + + +class RHEL(linux.LinuxPlatform): + """Red Hat Enterprise Linux specific information.""" + + @staticmethod + def IsThisPlatform(root='/'): + (distribution, _, _) = platform.linux_distribution() + if distribution == 'Red Hat Enterprise Linux Server': + return True + return False + + def __init__(self): + super(RHEL, self).__init__() + + def GetPreferredFilesystemType(self): + (_,version,_) = platform.linux_distribution() + if version.startswith('7'): + return 'xfs' + return 'ext4' diff --git a/gcimagebundle/gcimagebundlelib/sle.py b/gcimagebundle/gcimagebundlelib/sle.py new file mode 100644 index 0000000..8b74827 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/sle.py @@ -0,0 +1,34 @@ +# Copyright 2013 SUSE LLC All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""SUSE Linux Enterprise (SLE) platform info.""" + +import re +from gcimagebundlelib import suse + +class SLE(suse.SUSE): + """SLE platform info.""" + + @staticmethod + def IsThisPlatform(self, root='/'): + if re.match(r'SUSE Linux Enterprise', suse.SUSE().distribution): + return True + return False + + def __init__(self): + super(SLE, self).__init__() + + def GetPreferredFilesystemType(self): + return 'ext3' diff --git a/gcimagebundle/gcimagebundlelib/suse.py b/gcimagebundle/gcimagebundlelib/suse.py new file mode 100644 index 0000000..4911b8b --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/suse.py @@ -0,0 +1,91 @@ +# Copyright 2013 SUSE LLC All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""openSUSE and SUSE generic platform info.""" + +import os +import re + +from gcimagebundlelib import linux + + +class SUSE(linux.LinuxPlatform): + """openSUSE and SUSE generic platform info.""" + + def __init__(self): + super(SUSE, self).__init__() + self.distribution_codename = None + self.ParseOSRelease() + if not self.distribution: + self.ParseSUSERelease() + if not self.distribution: + self.distribution = '' + + def ParseOSRelease(self): + """Parse the /etc/os-release file.""" + release_file = '/etc/os-release' + if not os.path.isfile(release_file): + self.distribution = None + return + lines = open(release_file, 'r').readlines() + for ln in lines: + if not ln: + continue + if re.match(r'^NAME=', ln): + self.distribution = self.__getData(ln) + if re.match(r'^VERSION_ID=', ln): + self.distribution_version = self.__getData(ln) + if re.match(r'^VERSION=', ln): + data = self.__getData(ln) + self.distribution_codename = data.split('(')[-1][:-1] + return + + def ParseSUSERelease(self): + """Parse /etc/SuSE-release file.""" + release_file = '/etc/SuSE-release' + if not os.path.isfile(release_file): + self.distribution = None + return + lines = open(release_file, 'r').readlines() + prts = lines[0].split() + cnt = 0 + self.distribution = '' + if len(prts): + while 1: + item = prts[cnt] + if re.match('\d', item): + item = None + break + elif cnt > 0: + self.distribution += ' ' + self.distribution += item + cnt += 1 + + for ln in lines: + if re.match(r'^VERSION =', ln): + self.distribution_version = self.__getData(ln) + if re.match(r'^CODENAME =', ln): + self.distribution_codename = self.__getData(ln) + return + + def __getData(self, ln): + """Extract data from a line in a file. Either returns data inside the + first double quotes ("a b"; a b in this example) or if no double + quotes exist, returns the data after the first = sign. Leading + and trailing whitspace are stripped.""" + if ln.find('"') != -1: + return ln.split('"')[1] + else: + return ln.split('=')[-1].strip() diff --git a/gcimagebundle/gcimagebundlelib/tests/__init__.py b/gcimagebundle/gcimagebundlelib/tests/__init__.py new file mode 100644 index 0000000..42723d7 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/tests/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/python +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests module for Image Bundle.""" diff --git a/gcimagebundle/gcimagebundlelib/tests/block_disk_test.py b/gcimagebundle/gcimagebundlelib/tests/block_disk_test.py new file mode 100755 index 0000000..1cbb384 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/tests/block_disk_test.py @@ -0,0 +1,512 @@ +#!/usr/bin/python +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittest for block_disk.py module.""" + + +__pychecker__ = 'no-local' # for unittest + +from contextlib import closing +import json +import logging +import os +import random +import subprocess +import tarfile +import tempfile +import unittest +import urllib2 + +from gcimagebundlelib import block_disk +from gcimagebundlelib import exclude_spec +from gcimagebundlelib.tests import image_bundle_test_base +from gcimagebundlelib import utils + + +class FsRawDiskTest(image_bundle_test_base.ImageBundleTest): + """FsRawDisk Unit Test.""" + + _MEGABYTE = 1024*1024 + _GIGABYTE = 1024*_MEGABYTE + + def setUp(self): + super(FsRawDiskTest, self).setUp() + self._fs_size = 10* FsRawDiskTest._MEGABYTE + self._bundle = block_disk.FsRawDisk(self._fs_size, 'ext4') + self._tar_path = self.tmp_path + '/image.tar.gz' + self._bundle.SetTarfile(self._tar_path) + self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)]) + self._bundle.SetKey('key') + self._bundle._SetManifest(self._manifest) + + def _SetupMbrDisk(self, partition_start, partition_end, fs_uuid): + """Creates a disk with a fake MBR. + + Args: + partition_start: The byte offset where the partition starts. + partition_end: The byte offset where the partition ends. + fs_uuid: The UUID of the filesystem to create on the partition. + + Returns: + The path where the disk is located. + """ + # Create the disk file with the size specified. + disk_path = os.path.join(self.tmp_root, 'mbrdisk.raw') + disk_size = partition_end + FsRawDiskTest._MEGABYTE + with open(disk_path, 'wb') as disk_file: + disk_file.truncate(disk_size) + + # Create a partition table + utils.MakePartitionTable(disk_path) + + # Create the partition + utils.MakePartition(disk_path, 'primary', 'ext2', + partition_start, partition_end) + + # Create the file system + with utils.LoadDiskImage(disk_path) as devices: + utils.MakeFileSystem(devices[0], 'ext4', fs_uuid) + + # Write some data after the MBR but before the first partition + with open(disk_path, 'r+b') as disk_file: + # Seek to last two bytes of first sector + disk_file.seek(510) + # Write MBR signature + disk_file.write(chr(0x55)) + disk_file.write(chr(0xAA)) + # Write random data on the disk till the point first partition starts + for _ in range(partition_start - 512): + # Write a byte + disk_file.write(chr(random.randint(0, 127))) + + return disk_path + + def tearDown(self): + super(FsRawDiskTest, self).tearDown() + + def testDiskBundle(self): + """Tests bundle command when a disk is specified. + + Creates a 20Gb source disk to start with and verifies that creating + a 10MB file off it works. + """ + # Create a 20GB disk with first partition starting at 1MB + self._TestDiskBundleHelper(FsRawDiskTest._MEGABYTE, + FsRawDiskTest._GIGABYTE*20, + utils.RunCommand(['uuidgen']).strip()) + + def testDiskBundlePartitionAt2MB(self): + """Tests bundle command when a disk is specified. + + Creates the first partition at 2MB and verifies all data prior to that is + copied. + """ + # Create a 20GB disk with first partition starting at 2MB + self._TestDiskBundleHelper(FsRawDiskTest._MEGABYTE*2, + FsRawDiskTest._GIGABYTE*20, + utils.RunCommand(['uuidgen']).strip()) + + def _TestDiskBundleHelper(self, partition_start, partition_end, fs_uuid): + disk_path = self._SetupMbrDisk(partition_start, partition_end, fs_uuid) + + with utils.LoadDiskImage(disk_path) as devices: + # Get the path to do the disk. + # devices will have something which is like /dev/mapper/loop0p1 + # We need to get loop0 out of it. + disk_loop_back_path = '/dev/' + devices[0].split('/')[3][:-2] + + # Create a symlinks to the disk and loopback paths + # This is required because of the code where we assume first + # partition is device path appended by 1. Will remove it once we + # update that part of the code. + symlink_disk = os.path.join(self.tmp_root, 'disk') + symlink_partition = self.tmp_root + '/disk1' + utils.RunCommand(['ln', '-s', disk_loop_back_path, symlink_disk]) + utils.RunCommand(['ln', '-s', devices[0], symlink_partition]) + + # Bundle up + self._bundle.AddDisk(symlink_disk) + self._bundle.AddSource(self.tmp_path) + self._bundle.Verify() + (_, _) = self._bundle.Bundleup() + self._VerifyImageHas(self._tar_path, + ['lost+found', 'test1', 'test2', 'dir1/', + '/dir1/dir11/', '/dir1/sl1', '/dir1/hl2', 'dir2/', + '/dir2/dir1', '/dir2/sl2', '/dir2/hl1']) + self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test1', 2) + self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test2', 2) + self._VerifyDiskSize(self._tar_path, self._fs_size) + self._VerifyNonPartitionContents(self._tar_path, + disk_path, + partition_start) + self._VerifyFilesystemUUID(self._tar_path, fs_uuid) + + def testRawDisk(self): + """Tests the regular operation. No expected error.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + self._VerifyTarHas(self._tar_path, ['disk.raw']) + self._VerifyImageHas(self._tar_path, + ['lost+found', 'test1', 'test2', 'dir1/', + '/dir1/dir11/', '/dir1/sl1', '/dir1/hl2', 'dir2/', + '/dir2/dir1', '/dir2/sl2', '/dir2/hl1']) + self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test1', 2) + self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test2', 2) + + def testRawDiskIgnoresHardlinks(self): + """Tests if the raw disk ignores hard links if asked.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.IgnoreHardLinks() + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + self._VerifyTarHas(self._tar_path, ['disk.raw']) + self._VerifyImageHas(self._tar_path, + ['lost+found', 'test1', 'test2', 'dir1/', + '/dir1/dir11/', '/dir1/sl1', '/dir1/hl2', 'dir2/', + '/dir2/dir1', '/dir2/sl2', '/dir2/hl1']) + self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test1', 1) + self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test2', 1) + + def testRawDiskIgnoresExcludes(self): + """Tests if the raw disk ignores specified excludes files.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.AppendExcludes( + [exclude_spec.ExcludeSpec(self.tmp_path + '/dir1')]) + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + self._VerifyTarHas(self._tar_path, ['disk.raw']) + self._VerifyImageHas(self._tar_path, + ['lost+found', 'test1', 'test2', 'dir2/', '/dir2/dir1', + '/dir2/sl2', '/dir2/hl1']) + + def testRawDiskExcludePreservesSubdirs(self): + """Tests if excludes preserves subdirs underneath if asked.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.AppendExcludes( + [exclude_spec.ExcludeSpec(self.tmp_path + '/dir1', + preserve_dir=True, + preserve_subdir=True)]) + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + self._VerifyTarHas(self._tar_path, ['disk.raw']) + self._VerifyImageHas(self._tar_path, + ['lost+found', 'test1', 'test2', 'dir1/', + '/dir1/dir11', 'dir2/', '/dir2/dir1', + '/dir2/sl2', '/dir2/hl1']) + + def testRawDiskExcludePreservesFiles(self): + """Tests if excludes preserves the files underneath if asked.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.AppendExcludes( + [exclude_spec.ExcludeSpec(self.tmp_path + '/dir1', + preserve_dir=True, + preserve_file=True)]) + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + self._VerifyTarHas(self._tar_path, ['disk.raw']) + self._VerifyImageHas(self._tar_path, + ['lost+found', 'test1', 'test2', 'dir1/', '/dir1/hl2', + '/dir1/sl1', 'dir2/', '/dir2/dir1', '/dir2/sl2', + '/dir2/hl1']) + + def testRawDiskUsesModifiedFiles(self): + """Tests if the raw disk uses modified files.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.AppendExcludes( + [exclude_spec.ExcludeSpec(self.tmp_path + '/dir1')]) + self._bundle.SetPlatform(image_bundle_test_base.MockPlatform(self.tmp_root)) + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + self._VerifyTarHas(self._tar_path, ['disk.raw']) + self._VerifyImageHas(self._tar_path, + ['lost+found', 'test1', 'test2', 'dir2/', + '/dir2/dir1', '/dir2/sl2', '/dir2/hl1']) + self._VerifyFileInRawDiskEndsWith(self._tar_path, 'test1', + 'something extra.') + + def testRawDiskGeneratesCorrectDigest(self): + """Tests if the SHA1 digest generated is accurate.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + p = subprocess.Popen(['/usr/bin/openssl dgst -sha1 ' + self._tar_path], + stdout=subprocess.PIPE, shell=True) + file_digest = p.communicate()[0].split('=')[1].strip() + self.assertEqual(digest, file_digest) + + def testRawDiskHonorsRecursiveOff(self): + """Tests if raw disk handles recursive off.""" + self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)]) + self._bundle.AddSource(self.tmp_path + '/dir1', + arcname='dir1', recursive=False) + self._bundle.AddSource(self.tmp_path + '/dir2', arcname='dir2') + self._bundle.Verify() + (_, digest) = self._bundle.Bundleup() + if not digest: + self.fail('raw disk failed') + self._VerifyTarHas(self._tar_path, ['disk.raw']) + self._VerifyImageHas(self._tar_path, + ['lost+found', 'dir1/', 'dir2/', '/dir2/dir1', + '/dir2/sl2', '/dir2/hl1']) + + def testSkipLicenseCheckIfNotOnGCE(self): + """Tests that no licenses are loaded if gcimagebundle is not run on GCE.""" + class MockHttp(utils.Http): + def Get(self, request, timeout=None): + # if gcimagebundle is not run on GCE the metadata server will be unreachable + raise urllib2.URLError("urlopen error timed out") + + self._http = MockHttp() + self._manifest._http = self._http + self._manifest._is_gce_instance = False + + self._bundle.AddSource(self.tmp_path) + self._bundle.Verify() + _ = self._bundle.Bundleup() + self.assertFalse(self._bundle._manifest._IsManifestNeeded()) + self._VerifyTarHas(self._tar_path, ['disk.raw']) + + def testNoManifestCreatedWithZeroLicenses(self): + """Tests that no manifest is created when there are 0 licenses.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.Verify() + _ = self._bundle.Bundleup() + self.assertFalse(self._bundle._manifest._IsManifestNeeded()) + self._VerifyTarHas(self._tar_path, ['disk.raw']) + + def testManifestWithOneLicense(self): + """Tests manifest is populated with 1 license.""" + self._http._instance_response = ('{"hostname":"test",' + '"licenses":[{"id":"TEST-LICENSE"}]}') + self._bundle.AddSource(self.tmp_path) + self._bundle.Verify() + _ = self._bundle.Bundleup() + manifest_json = self._bundle._manifest._ToJson() + manifest_obj = json.loads(manifest_json) + self.assertTrue(self._bundle._manifest._IsManifestNeeded()) + self.assertEqual(1, len(manifest_obj['licenses'])) + self.assertEqual('TEST-LICENSE', manifest_obj['licenses'][0]) + self._VerifyTarHas(self._tar_path, ['manifest.json', 'disk.raw']) + self._VerifyFileContentsInTarball(self._tar_path, + 'manifest.json', + '{"licenses": ["TEST-LICENSE"]}') + + def testManifestWithTwoLicenses(self): + """Tests manifest is populated with 2 licenses.""" + self._http._instance_response = ('{"hostname":"test",' + '"licenses":[{"id":"TEST-1"},' + '{"id":"TEST-2"}]}') + self._bundle.AddSource(self.tmp_path) + self._bundle.Verify() + _ = self._bundle.Bundleup() + manifest_json = self._bundle._manifest._ToJson() + manifest_obj = json.loads(manifest_json) + self.assertTrue(self._bundle._manifest._IsManifestNeeded()) + self.assertEqual(2, len(manifest_obj['licenses'])) + self.assertEqual('TEST-1', manifest_obj['licenses'][0]) + self.assertEqual('TEST-2', manifest_obj['licenses'][1]) + self._VerifyTarHas(self._tar_path, ['manifest.json', 'disk.raw']) + self._VerifyFileContentsInTarball(self._tar_path, + 'manifest.json', + '{"licenses": ["TEST-1", "TEST-2"]}') + + def _VerifyFilesystemUUID(self, tar, expected_uuid): + """Verifies UUID of the first partition on disk matches the value.""" + tmp_dir = tempfile.mkdtemp(dir=self.tmp_root) + tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir] + self.assertEqual(subprocess.call(tar_cmd), 0) + + created_disk_path = os.path.join(tmp_dir, 'disk.raw') + with utils.LoadDiskImage(created_disk_path) as devices: + self.assertEqual(1, len(devices)) + self.assertEqual(expected_uuid, utils.GetUUID(devices[0])) + + def _VerifyNonPartitionContents(self, tar, disk_path, partition_start): + """Verifies that bytes outside the partition are preserved.""" + tmp_dir = tempfile.mkdtemp(dir=self.tmp_root) + tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir] + self.assertEqual(subprocess.call(tar_cmd), 0) + created_disk_path = os.path.join(tmp_dir, 'disk.raw') + + # Verify first parition in both disks starts at the same offset + self.assertEqual(partition_start, + utils.GetPartitionStart(disk_path, 1)) + self.assertEqual(partition_start, + utils.GetPartitionStart(created_disk_path, 1)) + with open(disk_path, 'r') as source_file: + with open(created_disk_path, 'r') as created_file: + # Seek to 510'th byte in both streams and verify rest of the + # bytes until the partition start are the same + source_file.seek(510) + created_file.seek(510) + for i in range(partition_start - 510): + self.assertEqual(source_file.read(1), + created_file.read(1), + 'byte at position %s not equal' % (i + 510)) + + def _VerifyDiskSize(self, tar, expected_size): + """Verifies that the disk file has the same size as expected.""" + tmp_dir = tempfile.mkdtemp(dir=self.tmp_root) + tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir] + self.assertEqual(subprocess.call(tar_cmd), 0) + disk_path = os.path.join(tmp_dir, 'disk.raw') + statinfo = os.stat(disk_path) + self.assertEqual(expected_size, statinfo.st_size) + + def _VerifyImageHas(self, tar, expected): + """Tests if raw disk contains an expected list of files/directories.""" + tmp_dir = tempfile.mkdtemp(dir=self.tmp_root) + tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir] + self.assertEqual(subprocess.call(tar_cmd), 0) + disk_path = os.path.join(tmp_dir, 'disk.raw') + with utils.LoadDiskImage(disk_path) as devices: + self.assertEqual(len(devices), 1) + mnt_dir = tempfile.mkdtemp(dir=self.tmp_root) + with utils.MountFileSystem(devices[0], mnt_dir, 'ext4'): + found = [] + for root, dirs, files in os.walk(mnt_dir): + root = root.replace(mnt_dir, '') + for f in files: + found.append(os.path.join(root, f)) + for d in dirs: + found.append(os.path.join(root, d)) + self._AssertListEqual(expected, found) + + def _VerifyFileContentsInTarball(self, tar, file_name, expected_content): + """Reads the file from the tar file and turns it.""" + with closing(tarfile.open(tar)) as tar_file: + content = tar_file.extractfile(file_name).read() + self.assertEqual(content, expected_content) + + def _VerifyFileInRawDiskEndsWith(self, tar, filename, text): + """Tests if a file on raw disk contains ends with a specified text.""" + tmp_dir = tempfile.mkdtemp(dir=self.tmp_root) + tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir] + self.assertEqual(subprocess.call(tar_cmd), 0) + disk_path = os.path.join(tmp_dir, 'disk.raw') + with utils.LoadDiskImage(disk_path) as devices: + self.assertEqual(len(devices), 1) + mnt_dir = tempfile.mkdtemp(dir=self.tmp_root) + with utils.MountFileSystem(devices[0], mnt_dir, 'ext4'): + f = open(os.path.join(mnt_dir, filename), 'r') + file_content = f.read() + f.close() + self.assertTrue(file_content.endswith(text)) + + def _VerifyNumberOfHardLinksInRawDisk(self, tar, filename, count): + """Tests if a file on raw disk has a specified number of hard links.""" + tmp_dir = tempfile.mkdtemp(dir=self.tmp_root) + tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir] + self.assertEqual(subprocess.call(tar_cmd), 0) + disk_path = os.path.join(tmp_dir, 'disk.raw') + with utils.LoadDiskImage(disk_path) as devices: + self.assertEqual(len(devices), 1) + mnt_dir = tempfile.mkdtemp(dir=self.tmp_root) + with utils.MountFileSystem(devices[0], mnt_dir, 'ext4'): + self.assertEqual(os.stat(os.path.join(mnt_dir, filename)).st_nlink, + count) + + +class RootFsRawTest(image_bundle_test_base.ImageBundleTest): + """RootFsRaw Unit Test.""" + + def setUp(self): + super(RootFsRawTest, self).setUp() + self._bundle = block_disk.RootFsRaw( + 10*1024*1024, 'ext4', False, self._MockStatvfs) + self._tar_path = self.tmp_path + '/image.tar.gz' + self._bundle.SetTarfile(self._tar_path) + self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)]) + self._bundle._SetManifest(self._manifest) + + def tearDown(self): + super(RootFsRawTest, self).tearDown() + + def testRootRawDiskVerifiesOneSource(self): + """Tests that only one root directory is allowed.""" + self._bundle.AddSource(self.tmp_path) + self._bundle.AddSource(self.tmp_path + '/dir1') + self._bundle.SetKey('key') + try: + self._bundle.Verify() + except block_disk.InvalidRawDiskError: + return + self.fail() + + def testRootRawDiskVerifiesRootDestination(self): + """Tests that destination directory must be /.""" + self._bundle.AddSource(self.tmp_path, arcname='/tmp') + self._bundle.SetKey('key') + try: + self._bundle.Verify() + except block_disk.InvalidRawDiskError: + return + self.fail() + + def testRootRawDiskNotEnoughFreeSpace(self): + """Tests that there is not enough disk space to complete the operation.""" + self._statvfs_map = { + "/" : image_bundle_test_base.StatvfsResult(1024, 500, 100), + "/tmp" : image_bundle_test_base.StatvfsResult(1024, 500, 100) + } + self._bundle.AddSource("/") + self._bundle.SetKey('key') + try: + self._bundle.Verify() + except block_disk.InvalidRawDiskError as e: + print str(e) + return + self.fail() + + def testRootFilesExceedDiskSize(self): + """Tests that source files may exceed the raw disk file size limit.""" + self._statvfs_map = { + "/" : image_bundle_test_base.StatvfsResult(1024, 50000, 20000), + "/tmp" : image_bundle_test_base.StatvfsResult(1024, 100000, 90000) + } + self._bundle.AddSource("/") + self._bundle.SetKey('key') + try: + self._bundle.Verify() + except block_disk.InvalidRawDiskError as e: + print str(e) + return + self.fail() + + def _MockStatvfs(self, file_path): + return self._statvfs_map[file_path] + +def main(): + logging.basicConfig(level=logging.DEBUG) + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/gcimagebundle/gcimagebundlelib/tests/image_bundle_test_base.py b/gcimagebundle/gcimagebundlelib/tests/image_bundle_test_base.py new file mode 100755 index 0000000..37b7fae --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/tests/image_bundle_test_base.py @@ -0,0 +1,140 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Base class for image_bundle unittests.""" + + +__pychecker__ = 'no-local' # for unittest + + +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +import unittest +import urllib2 + +from gcimagebundlelib import manifest +from gcimagebundlelib.os_platform import Platform +from gcimagebundlelib import utils + + +class InvalidOverwriteFileException(Exception): + """Invalid overwrite target was passed to MockPlatform.Overwrite method.""" + + +class MockPlatform(Platform): + """Mock platform for image bundle unit tests.""" + OVERWRITE_LIST = ['test1'] + + def __init__(self, tmp_root): + super(MockPlatform, self).__init__() + self.tmp_root = tmp_root + + def Overwrite(self, filename, arcname, tmpdir): + temp = tempfile.mktemp(dir=tmpdir) + if arcname != 'test1': + raise InvalidOverwriteFileException(arcname) + fd = open(temp, 'w') + fd.write(open(filename).read()) + fd.write('something extra.') + fd.close() + return temp + + +class MockHttp(utils.Http): + """Fake implementation of the utils.Http client. Used for metadata queries.""" + def __init__(self): + self._instance_response = '{"hostname":"test"}' + + def Get(self, request, timeout=None): + """Accepts an Http request and returns a precanned response.""" + url = request.get_full_url() + if url == utils.METADATA_URL_PREFIX: + return 'v1/' + elif url.startswith(utils.METADATA_V1_URL_PREFIX): + url = url.replace(utils.METADATA_V1_URL_PREFIX, '') + if url == 'instance/?recursive=true': + return self._instance_response + raise urllib2.HTTPError + +class StatvfsResult: + """ A struct for partial os.statvfs result, used to mock the result. """ + + def __init__(self, f_bsize, f_blocks, f_bfree): + self.f_bsize = f_bsize + self.f_blocks = f_blocks + self.f_bfree = f_bfree + +class ImageBundleTest(unittest.TestCase): + """ImageBundle Unit Test Base Class.""" + + def setUp(self): + self.tmp_root = tempfile.mkdtemp(dir='/tmp') + self.tmp_path = tempfile.mkdtemp(dir=self.tmp_root) + self._http = MockHttp() + self._manifest = manifest.ImageManifest(http=self._http, is_gce_instance=True) + self._SetupFilesystemToTar() + + def tearDown(self): + self._CleanupFiles() + + def _SetupFilesystemToTar(self): + """Creates some directory structure to tar.""" + if os.path.exists(self.tmp_path): + shutil.rmtree(self.tmp_path) + os.makedirs(self.tmp_path) + with open(self.tmp_path + '/test1', 'w') as fd: + print >> fd, 'some text' + shutil.copyfile(self.tmp_path + '/test1', self.tmp_path + '/test2') + os.makedirs(self.tmp_path + '/dir1') + os.makedirs(self.tmp_path + '/dir1/dir11') + os.makedirs(self.tmp_path + '/dir2') + os.makedirs(self.tmp_path + '/dir2/dir1') + os.symlink(self.tmp_path + '/test1', self.tmp_path + '/dir1/sl1') + os.link(self.tmp_path + '/test2', self.tmp_path + '/dir1/hl2') + os.symlink(self.tmp_path + '/test2', self.tmp_path + '/dir2/sl2') + os.link(self.tmp_path + '/test1', self.tmp_path + '/dir2/hl1') + + def _CleanupFiles(self): + """Removes the files under test directory.""" + if os.path.exists(self.tmp_root): + shutil.rmtree(self.tmp_root) + + def _VerifyTarHas(self, tar, expected): + p = subprocess.Popen(['tar -tf %s' % tar], + stdout=subprocess.PIPE, shell=True) + found = p.communicate()[0].split('\n') + if './' in found: + found.remove('./') + if '' in found: + found.remove('') + self._AssertListEqual(expected, found) + + def _VerifyFileInTarEndsWith(self, tar, filename, text): + tf = tarfile.open(tar, 'r:gz') + fd = tf.extractfile(filename) + file_content = fd.read() + self.assertTrue(file_content.endswith(text)) + + def _AssertListEqual(self, list1, list2): + """Asserts that, when sorted, list1 and list2 are identical.""" + sorted_list1 = [re.sub(r'/$', '', x) for x in list1] + sorted_list2 = [re.sub(r'/$', '', x) for x in list2] + sorted_list1.sort() + sorted_list2.sort() + self.assertEqual(sorted_list1, sorted_list2) diff --git a/gcimagebundle/gcimagebundlelib/tests/utils_test.py b/gcimagebundle/gcimagebundlelib/tests/utils_test.py new file mode 100755 index 0000000..dd7d2cd --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/tests/utils_test.py @@ -0,0 +1,49 @@ +#!/usr/bin/python +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittest for utils.py module.""" + +__pychecker__ = 'no-local' # for unittest + +import logging +import subprocess +import unittest +import uuid + +from gcimagebundlelib import utils + + +class ImageBundleTest(unittest.TestCase): + + def testRunCommand(self): + """Run a simple command and verify it works.""" + utils.RunCommand(['ls', '/']) + + def testRunCommandThatFails(self): + """Run a command that will fail and verify it raises the correct error.""" + def RunCommandUnderTest(): + non_existent_path = '/' + uuid.uuid4().hex + utils.RunCommand(['mkfs', '-t', 'ext4', non_existent_path]) + self.assertRaises(subprocess.CalledProcessError, RunCommandUnderTest) + + +def main(): + logging.basicConfig(level=logging.DEBUG) + unittest.main() + + +if __name__ == '__main__': + main() + diff --git a/gcimagebundle/gcimagebundlelib/ubuntu.py b/gcimagebundle/gcimagebundlelib/ubuntu.py new file mode 100644 index 0000000..8d68687 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/ubuntu.py @@ -0,0 +1,54 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ubuntu specific platform info.""" + +import csv +import os +from gcimagebundlelib import linux + + +class Ubuntu(linux.LinuxPlatform): + """Ubuntu specific information.""" + + @staticmethod + def IsThisPlatform(root='/'): + release_file = root + '/etc/lsb-release' + if os.path.exists(release_file): + (_, _, flavor, _) = Ubuntu.ParseLsbRelease(release_file) + if flavor and flavor.lower() == 'ubuntu': + return True + return False + + @staticmethod + def ParseLsbRelease(release_file='/etc/lsb-release'): + """Parses the /etc/lsb-releases file.""" + release_info = {} + for line in csv.reader(open(release_file), delimiter='='): + if len(line) > 1: + release_info[line[0]] = line[1] + if ('DISTRIB_CODENAME' not in release_info or + 'DISTRIB_DESCRIPTION' not in release_info or + 'DISTRIB_ID' not in release_info or + 'DISTRIB_RELEASE' not in release_info): + return (None, None, None, None) + return (release_info['DISTRIB_CODENAME'], + release_info['DISTRIB_DESCRIPTION'], + release_info['DISTRIB_ID'], + release_info['DISTRIB_RELEASE']) + + def __init__(self): + super(Ubuntu, self).__init__() + (self.distribution_codename, _, self.distribution, + self.distribution_version) = Ubuntu.ParseLsbRelease() diff --git a/gcimagebundle/gcimagebundlelib/utils.py b/gcimagebundle/gcimagebundlelib/utils.py new file mode 100644 index 0000000..a8fde40 --- /dev/null +++ b/gcimagebundle/gcimagebundlelib/utils.py @@ -0,0 +1,455 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Utilities for image bundling tool.""" + +import logging +import os +import subprocess +import time +import urllib2 + +METADATA_URL_PREFIX = 'http://169.254.169.254/computeMetadata/' +METADATA_V1_URL_PREFIX = METADATA_URL_PREFIX + 'v1/' + + +class MakeFileSystemException(Exception): + """Error occurred in file system creation.""" + + +class TarAndGzipFileException(Exception): + """Error occurred in creating the tarball.""" + + +class LoadDiskImage(object): + """Loads raw disk image using kpartx.""" + + def __init__(self, file_path): + """Initializes LoadDiskImage object. + + Args: + file_path: a path to a file containing raw disk image. + + Returns: + A list of devices for every partition found in an image. + """ + self._file_path = file_path + + def __enter__(self): + """Map disk image as a device.""" + SyncFileSystem() + kpartx_cmd = ['kpartx', '-a', '-v', '-s', self._file_path] + output = RunCommand(kpartx_cmd) + devs = [] + for line in output.splitlines(): + split_line = line.split() + if (len(split_line) > 2 and split_line[0] == 'add' + and split_line[1] == 'map'): + devs.append('/dev/mapper/' + split_line[2]) + time.sleep(2) + return devs + + def __exit__(self, unused_exc_type, unused_exc_value, unused_exc_tb): + """Unmap disk image as a device. + + Args: + unused_exc_type: unused. + unused_exc_value: unused. + unused_exc_tb: unused. + """ + SyncFileSystem() + time.sleep(2) + kpartx_cmd = ['kpartx', '-d', '-v', '-s', self._file_path] + RunCommand(kpartx_cmd) + + +class MountFileSystem(object): + """Mounts a file system.""" + + def __init__(self, dev_path, dir_path, fs_type): + """Initializes MountFileSystem object. + + Args: + dev_path: A path to a device to mount. + dir_path: A path to a directory where a device is to be mounted. + """ + self._dev_path = dev_path + self._dir_path = dir_path + self._fs_type = fs_type + + def __enter__(self): + """Mounts a device. + """ + # Since the bundled image can have the same uuid as the root disk, + # we should prevent uuid conflicts for xfs mounts. + if self._fs_type is 'xfs': + mount_cmd = ['mount', '-o', 'nouuid', self._dev_path, self._dir_path] + else: + mount_cmd = ['mount', self._dev_path, self._dir_path] + RunCommand(mount_cmd) + + def __exit__(self, unused_exc_type, unused_exc_value, unused_exc_tb): + """Unmounts a file system. + + Args: + unused_exc_type: unused. + unused_exc_value: unused. + unused_exc_tb: unused. + """ + umount_cmd = ['umount', self._dir_path] + RunCommand(umount_cmd) + SyncFileSystem() + + +def SyncFileSystem(): + RunCommand(['sync']) + +def GetMounts(root='/'): + """Find all mount points under the specified root. + + Args: + root: a path to look for a mount points. + + Returns: + A list of mount points. + """ + output = RunCommand(['/bin/mount', '-l']) + mounts = [] + for line in output.splitlines(): + split_line = line.split() + mount_point = split_line[2] + if mount_point == root: + continue + # We are simply ignoring the fs_type of fs for now. But we can use that + # later Just verify that these are actually mount points. + if os.path.ismount(mount_point) and mount_point.startswith(root): + mounts.append(mount_point) + return mounts + + +def MakePartitionTable(file_path): + """Create a partition table in a file. + + Args: + file_path: A path to a file where a partition table will be created. + """ + RunCommand(['parted', file_path, 'mklabel', 'msdos']) + + +def MakePartition(file_path, partition_type, fs_type, start, end): + """Create a partition in a file. + + Args: + file_path: A path to a file where a partition will be created. + partition_type: A type of a partition to be created. Tested option is msdos. + fs_type: A type of a file system to be created. For example, ext2, ext3, + etc. + start: Start offset of a partition in bytes. + end: End offset of a partition in bytes. + """ + parted_cmd = ['parted', file_path, 'unit B', 'mkpart', partition_type, + fs_type, str(start), str(end)] + RunCommand(parted_cmd) + + +def MakeFileSystem(dev_path, fs_type, uuid=None): + """Create a file system in a device. + + Args: + dev_path: A path to a device. + fs_type: A type of a file system to be created. For example ext2, ext3, etc. + uuid: The value to use as the UUID for the filesystem. If none, a random + UUID will be generated and used. + + Returns: + The uuid of the filesystem. This will be the same as the passed value if + a value was specified. If no uuid was passed in, this will be the randomly + generated uuid. + + Raises: + MakeFileSystemException: If mkfs encounters an error. + """ + if uuid is None: + uuid = RunCommand(['uuidgen']).strip() + if uuid is None: + raise MakeFileSystemException(dev_path) + + mkfs_cmd = ['mkfs', '-t', fs_type, dev_path] + RunCommand(mkfs_cmd) + + if fs_type is 'xfs': + set_uuid_cmd = ['xfs_admin', '-U', uuid, dev_path] + else: + set_uuid_cmd = ['tune2fs', '-U', uuid, dev_path] + RunCommand(set_uuid_cmd) + + return uuid + + +def Rsync(src, dest, exclude_file, ignore_hard_links, recursive, xattrs): + """Copy files from specified directory using rsync. + + Args: + src: Source location to copy. + dest: Destination to copy files to. + exclude_file: A path to a file which contains a list of exclude from copy + filters. + ignore_hard_links: If True a hard links are copied as a separate files. If + False, hard link are recreated in dest. + recursive: Specifies if directories are copied recursively or not. + xattrs: Specifies if extended attributes are preserved or not. + """ + rsync_cmd = ['rsync', '--times', '--perms', '--owner', '--group', '--links', + '--devices', '--acls', '--sparse'] + if not ignore_hard_links: + rsync_cmd.append('--hard-links') + if recursive: + rsync_cmd.append('--recursive') + else: + rsync_cmd.append('--dirs') + if xattrs: + rsync_cmd.append('--xattrs') + if exclude_file: + rsync_cmd.append('--exclude-from=' + exclude_file) + rsync_cmd.extend([src, dest]) + + logging.debug('Calling: %s', repr(rsync_cmd)) + if exclude_file: + logging.debug('Contents of exclude file %s:', exclude_file) + with open(exclude_file, 'rb') as excludes: + for line in excludes: + logging.debug(' %s', line.rstrip()) + + RunCommand(rsync_cmd) + + +def GetUUID(partition_path): + """Fetches the UUID of the filesystem on the specified partition. + + Args: + partition_path: The path to the partition. + + Returns: + The uuid of the filesystem. + """ + output = RunCommand(['blkid', partition_path]) + for token in output.split(): + if token.startswith('UUID='): + uuid = token.strip()[len('UUID="'):-1] + + logging.debug('found uuid = %s', uuid) + return uuid + + +def CopyBytes(src, dest, count): + """Copies count bytes from the src to dest file. + + Args: + src: The source to read bytes from. + dest: The destination to copy bytes to. + count: Number of bytes to copy. + """ + block_size = 4096 + block_count = count / block_size + dd_command = ['dd', + 'if=%s' % src, + 'of=%s' % dest, + 'conv=notrunc', + 'bs=%s' % block_size, + 'count=%s' % block_count] + RunCommand(dd_command) + remaining_bytes = count - block_count * block_size + if remaining_bytes: + logging.debug('remaining bytes to copy = %s', remaining_bytes) + dd_command = ['dd', + 'if=%s' % src, + 'of=%s' % dest, + 'seek=%s' % block_count, + 'skip=%s' % block_count, + 'conv=notrunc', + 'bs=1', + 'count=%s' % remaining_bytes] + RunCommand(dd_command) + + +def GetPartitionStart(disk_path, partition_number): + """Returns the starting position in bytes of the partition. + + Args: + disk_path: The path to disk device. + partition_number: The partition number to lookup. 1 based. + + Returns: + The starting position of the first partition in bytes. + + Raises: + subprocess.CalledProcessError: If running parted fails. + IndexError: If there is no partition at the given number. + """ + parted_cmd = ['parted', + disk_path, + 'unit B', + 'print'] + # In case the device is not valid and parted throws the retry/cancel prompt + # write c to stdin. + output = RunCommand(parted_cmd, input_str='c') + for line in output.splitlines(): + split_line = line.split() + if len(split_line) > 4 and split_line[0] == str(partition_number): + return int(split_line[1][:-1]) + raise IndexError() + + +def RemovePartition(disk_path, partition_number): + """Removes the partition number from the disk. + + Args: + disk_path: The disk to remove the partition from. + partition_number: The partition number to remove. + """ + parted_cmd = ['parted', + disk_path, + 'rm', + str(partition_number)] + # In case the device is not valid and parted throws the retry/cancel prompt + # write c to stdin. + RunCommand(parted_cmd, input_str='c') + + +def GetDiskSize(disk_file): + """Returns the size of the disk device in bytes. + + Args: + disk_file: The full path to the disk device. + + Returns: + The size of the disk device in bytes. + + Raises: + subprocess.CalledProcessError: If fdisk command fails for the disk file. + """ + output = RunCommand(['fdisk', '-s', disk_file]) + return int(output) * 1024 + + +def RunCommand(command, input_str=None): + """Runs the command and returns the output printed on stdout. + + Args: + command: The command to run. + input_str: The input to pass to subprocess via stdin. + + Returns: + The stdout from running the command. + + Raises: + subprocess.CalledProcessError: if the command fails. + """ + logging.debug('running %s with input=%s', command, input_str) + p = subprocess.Popen(command, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd_output = p.communicate(input_str) + logging.debug('stdout %s', cmd_output[0]) + logging.debug('stderr %s', cmd_output[1]) + logging.debug('returncode %s', p.returncode) + if p.returncode: + logging.warning('Error while running %s return_code = %s\n' + 'stdout=%s\nstderr=%s', + command, p.returncode, cmd_output[0], + cmd_output[1]) + raise subprocess.CalledProcessError(p.returncode, + cmd=command) + return cmd_output[0] + + +def TarAndGzipFile(src_paths, dest): + """Pack file in tar archive and optionally gzip it. + + Args: + src_paths: A list of files that will be archived. + (Must be in the same directory.) + dest: An archive name. If a file ends with .gz or .tgz an archive is gzipped + as well. + + Raises: + TarAndGzipFileException: If tar encounters an error. + """ + if dest.endswith('.gz') or dest.endswith('.tgz'): + mode = 'czSf' + else: + mode = 'cSf' + src_names = [os.path.basename(src_path) for src_path in src_paths] + # Take the directory of the first file in the list, all files are expected + # to be in the same directory. + src_dir = os.path.dirname(src_paths[0]) + tar_cmd = ['tar', mode, dest, '-C', src_dir] + src_names + retcode = subprocess.call(tar_cmd) + if retcode: + raise TarAndGzipFileException(','.join(src_paths)) + + +class Http(object): + def Get(self, request, timeout=None): + return urllib2.urlopen(request, timeout=timeout).read() + + def GetMetadata(self, url_path, recursive=False, timeout=None): + """Retrieves instance metadata. + + Args: + url_path: The path of the metadata url after the api version. + http://169.254.169.254/computeMetadata/v1/url_path + recursive: If set, returns the tree of metadata starting at url_path as + a json string. + timeout: How long to wait for blocking operations (in seconds). + A value of None uses urllib2's default timeout. + Returns: + The metadata returned based on the url path. + + """ + # Use the latest version of the metadata. + suffix = '' + if recursive: + suffix = '?recursive=true' + url = '{0}{1}{2}'.format(METADATA_V1_URL_PREFIX, url_path, suffix) + request = urllib2.Request(url) + request.add_unredirected_header('Metadata-Flavor', 'Google') + return self.Get(request, timeout=timeout) + + +def IsRunningOnGCE(): + """Detect if we are running on GCE. + + Returns: + True if we are running on GCE, False otherwise. + """ + # Try accessing DMI/SMBIOS informations through dmidecode first + try: + dmidecode_cmd = ['dmidecode', '-s', 'bios-vendor'] + output = RunCommand(dmidecode_cmd) + return 'Google' in output + except subprocess.CalledProcessError: + # We fail if dmidecode doesn't exist or we have insufficient privileges + pass + + # If dmidecode is not working, fallback to contacting the metadata server + try: + Http().GetMetadata('instance/id', timeout=1) + return True + except urllib2.HTTPError as e: + logging.warning('HTTP error: %s (http status code=%s)' % (e.reason, e.code)) + except urllib2.URLError as e: + logging.warning('Cannot reach metadata server: %s' % e.reason) + + return False diff --git a/gcimagebundle/setup.py b/gcimagebundle/setup.py new file mode 100755 index 0000000..76ccd04 --- /dev/null +++ b/gcimagebundle/setup.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup installation module for Image Bundle.""" + +import os +import distribute_setup +distribute_setup.use_setuptools() + +from setuptools import find_packages +from setuptools import setup + +CURDIR = os.path.abspath(os.path.dirname(__file__)) + +def Read(file_name): + with open(os.path.join(CURDIR, file_name), 'r') as f: + return f.read().strip() + +setup( + name='gcimagebundle', + version=Read('VERSION'), + url='https://github.com/GoogleCloudPlatform/compute-image-packages/tree/master/image-bundle', + download_url='https://github.com/GoogleCloudPlatform/compute-image-packages/releases', + license='Apache 2.0', + author='Google Inc.', + author_email='gc-team@google.com', + description=('Image bundling tool for root file system.'), + long_description=Read('README.md'), + zip_safe=False, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Topic :: System :: Filesystems', + 'Topic :: Utilities', + ], + platforms='any', + include_package_data=True, + packages=find_packages(exclude=['distribute_setup']), + scripts=['gcimagebundle'], + test_suite='gcimagebundlelib.tests', +) diff --git a/gcimagebundle/stdeb.cfg b/gcimagebundle/stdeb.cfg new file mode 100644 index 0000000..09364a3 --- /dev/null +++ b/gcimagebundle/stdeb.cfg @@ -0,0 +1,3 @@ +[DEFAULT] +Depends: kpartx, parted, rsync, uuid-runtime +XS-Python-Version: >= 2.6 |