diff options
author | Konstantin Tokarev <annulen@yandex.ru> | 2016-08-25 19:20:41 +0300 |
---|---|---|
committer | Konstantin Tokarev <annulen@yandex.ru> | 2017-02-02 12:30:55 +0000 |
commit | 6882a04fb36642862b11efe514251d32070c3d65 (patch) | |
tree | b7959826000b061fd5ccc7512035c7478742f7b0 /Tools/Scripts/webkitpy/common/system/autoinstall.py | |
parent | ab6df191029eeeb0b0f16f127d553265659f739e (diff) | |
download | qtwebkit-6882a04fb36642862b11efe514251d32070c3d65.tar.gz |
Imported QtWebKit TP3 (git b57bc6801f1876c3220d5a4bfea33d620d477443)
Change-Id: I3b1d8a2808782c9f34d50240000e20cb38d3680f
Reviewed-by: Konstantin Tokarev <annulen@yandex.ru>
Diffstat (limited to 'Tools/Scripts/webkitpy/common/system/autoinstall.py')
-rw-r--r-- | Tools/Scripts/webkitpy/common/system/autoinstall.py | 433 |
1 files changed, 0 insertions, 433 deletions
diff --git a/Tools/Scripts/webkitpy/common/system/autoinstall.py b/Tools/Scripts/webkitpy/common/system/autoinstall.py deleted file mode 100644 index 2e15887bb..000000000 --- a/Tools/Scripts/webkitpy/common/system/autoinstall.py +++ /dev/null @@ -1,433 +0,0 @@ -# Copyright (c) 2009, Daniel Krech All rights reserved. -# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the Daniel Krech nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Support for automatically downloading Python packages from an URL.""" - - -import codecs -import logging -import os -import shutil -import stat -import sys -import tarfile -import tempfile -import urllib2 -import urlparse -import zipfile - -_log = logging.getLogger(__name__) - - -class AutoInstaller(object): - - """Supports automatically installing Python packages from an URL. - - Supports uncompressed files, .tar.gz, and .zip formats. - - Basic usage: - - installer = AutoInstaller() - - installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", - url_subpath="pep8-0.5.0/pep8.py") - installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.4.zip", - url_subpath="mechanize") - - """ - - def __init__(self, append_to_search_path=False, make_package=True, - target_dir=None, temp_dir=None): - """Create an AutoInstaller instance, and set up the target directory. - - Args: - append_to_search_path: A boolean value of whether to append the - target directory to the sys.path search path. - make_package: A boolean value of whether to make the target - directory a package. This adds an __init__.py file - to the target directory -- allowing packages and - modules within the target directory to be imported - explicitly using dotted module names. - target_dir: The directory path to which packages should be installed. - Defaults to a subdirectory of the folder containing - this module called "autoinstalled". - temp_dir: The directory path to use for any temporary files - generated while downloading, unzipping, and extracting - packages to install. Defaults to a standard temporary - location generated by the tempfile module. This - parameter should normally be used only for development - testing. - - """ - if target_dir is None: - this_dir = os.path.dirname(__file__) - target_dir = os.path.join(this_dir, "autoinstalled") - - # Ensure that the target directory exists. - self._set_up_target_dir(target_dir, append_to_search_path, make_package) - - self._target_dir = target_dir - self._temp_dir = temp_dir - - def _write_file(self, path, text, encoding): - with codecs.open(path, "w", encoding) as filehandle: - filehandle.write(text) - - def _set_up_target_dir(self, target_dir, append_to_search_path, - make_package): - """Set up a target directory. - - Args: - target_dir: The path to the target directory to set up. - append_to_search_path: A boolean value of whether to append the - target directory to the sys.path search path. - make_package: A boolean value of whether to make the target - directory a package. This adds an __init__.py file - to the target directory -- allowing packages and - modules within the target directory to be imported - explicitly using dotted module names. - - """ - if not os.path.exists(target_dir): - os.makedirs(target_dir) - - if append_to_search_path: - sys.path.append(target_dir) - - if make_package: - self._make_package(target_dir) - - def _make_package(self, target_dir): - init_path = os.path.join(target_dir, "__init__.py") - if not os.path.exists(init_path): - text = ("# This file is required for Python to search this " - "directory for modules.\n") - self._write_file(init_path, text, "ascii") - - def _create_scratch_directory_inner(self, prefix): - """Create a scratch directory without exception handling. - - Creates a scratch directory inside the AutoInstaller temp - directory self._temp_dir, or inside a platform-dependent temp - directory if self._temp_dir is None. Returns the path to the - created scratch directory. - - Raises: - OSError: [Errno 2] if the containing temp directory self._temp_dir - is not None and does not exist. - - """ - # The tempfile.mkdtemp() method function requires that the - # directory corresponding to the "dir" parameter already exist - # if it is not None. - scratch_dir = tempfile.mkdtemp(prefix=prefix.replace('/', '.'), dir=self._temp_dir) - return scratch_dir - - def _create_scratch_directory(self, target_name): - """Create a temporary scratch directory, and return its path. - - The scratch directory is generated inside the temp directory - of this AutoInstaller instance. This method also creates the - temp directory if it does not already exist. - - """ - prefix = target_name.replace(os.sep, "_") + "_" - try: - scratch_dir = self._create_scratch_directory_inner(prefix) - except OSError: - # Handle case of containing temp directory not existing-- - # OSError: [Errno 2] No such file or directory:... - temp_dir = self._temp_dir - if temp_dir is None or os.path.exists(temp_dir): - raise - # Else try again after creating the temp directory. - os.makedirs(temp_dir) - scratch_dir = self._create_scratch_directory_inner(prefix) - - return scratch_dir - - def _url_downloaded_path(self, target_name): - return os.path.join(self._target_dir, ".%s.url" % target_name.replace('/', '_')) - - def _is_downloaded(self, target_name, url): - version_path = self._url_downloaded_path(target_name) - - if not os.path.exists(version_path): - return False - - with codecs.open(version_path, "r", "utf-8") as filehandle: - return filehandle.read().strip() == url.strip() - - def _record_url_downloaded(self, target_name, url): - version_path = self._url_downloaded_path(target_name) - self._write_file(version_path, url, "utf-8") - - def _extract_targz(self, path, scratch_dir): - # tarfile.extractall() extracts to a path without the trailing ".tar.gz". - target_basename = os.path.basename(path[:-len(".tar.gz")]) - target_path = os.path.join(scratch_dir, target_basename) - - try: - tar_file = tarfile.open(path) - except tarfile.ReadError, err: - # Append existing Error message to new Error. - message = ("Could not open tar file: %s\n" - " The file probably does not have the correct format.\n" - " --> Inner message: %s" - % (path, err)) - raise Exception(message) - - try: - tar_file.extractall(target_path) - finally: - tar_file.close() - - return target_path - - # This is a replacement for ZipFile.extractall(), which is - # available in Python 2.6 but not in earlier versions. - # NOTE: The version in 2.6.1 (which shipped on Snow Leopard) is broken! - def _extract_all(self, zip_file, target_dir): - for name in zip_file.namelist(): - path = os.path.join(target_dir, name) - if not os.path.basename(path): - # Then the path ends in a slash, so it is a directory. - os.makedirs(path) - continue - - try: - # We open this file w/o encoding, as we're reading/writing - # the raw byte-stream from the zip file. - outfile = open(path, 'wb') - except IOError: - # Not all zip files seem to list the directories explicitly, - # so try again after creating the containing directory. - _log.debug("Got IOError: retrying after creating directory...") - dirname = os.path.dirname(path) - os.makedirs(dirname) - outfile = open(path, 'wb') - - try: - outfile.write(zip_file.read(name)) - finally: - outfile.close() - - def _unzip(self, path, scratch_dir): - # zipfile.extractall() extracts to a path without the trailing ".zip". - target_basename = os.path.basename(path[:-len(".zip")]) - target_path = os.path.join(scratch_dir, target_basename) - - try: - zip_file = zipfile.ZipFile(path, "r") - except zipfile.BadZipfile, err: - message = ("Could not open zip file: %s\n" - " --> Inner message: %s" - % (path, err)) - raise Exception(message) - - try: - self._extract_all(zip_file, scratch_dir) - finally: - zip_file.close() - - return target_path - - def _prepare_package(self, path, scratch_dir): - """Prepare a package for use, if necessary, and return the new path. - - For example, this method unzips zipped files and extracts - tar files. - - Args: - path: The path to the downloaded URL contents. - scratch_dir: The scratch directory. Note that the scratch - directory contains the file designated by the - path parameter. - - """ - # FIXME: Add other natural extensions. - if path.endswith(".zip"): - new_path = self._unzip(path, scratch_dir) - elif path.endswith(".tar.gz"): - new_path = self._extract_targz(path, scratch_dir) - else: - # No preparation is needed. - new_path = path - - return new_path - - def _download_to_stream(self, url, stream): - failures = 0 - while True: - try: - netstream = urllib2.urlopen(url) - break - except IOError, err: - # Try multiple times - if failures < 5: - _log.warning("Failed to download %s, %s retrying" % ( - url, err)) - failures += 1 - continue - - # Append existing Error message to new Error. - message = ('Could not download Python modules from URL "%s".\n' - " Make sure you are connected to the internet.\n" - " You must be connected to the internet when " - "downloading needed modules for the first time.\n" - " --> Inner message: %s" - % (url, err)) - raise IOError(message) - code = 200 - if hasattr(netstream, "getcode"): - code = netstream.getcode() - if not 200 <= code < 300: - raise ValueError("HTTP Error code %s" % code) - - BUFSIZE = 2**13 # 8KB - while True: - data = netstream.read(BUFSIZE) - if not data: - break - stream.write(data) - netstream.close() - - def _download(self, url, scratch_dir): - url_path = urlparse.urlsplit(url)[2] - url_path = os.path.normpath(url_path) # Removes trailing slash. - target_filename = os.path.basename(url_path) - target_path = os.path.join(scratch_dir, target_filename) - - with open(target_path, "wb") as stream: - self._download_to_stream(url, stream) - - return target_path - - def _install(self, scratch_dir, package_name, target_path, url, url_subpath, files_to_remove): - """Install a python package from an URL. - - This internal method overwrites the target path if the target - path already exists. - - """ - path = self._download(url=url, scratch_dir=scratch_dir) - path = self._prepare_package(path, scratch_dir) - - if url_subpath is None: - source_path = path - else: - source_path = os.path.join(path, url_subpath) - - for filename in files_to_remove: - path = os.path.join(source_path, filename.replace('/', os.sep)) - if os.path.exists(path): - # Pre-emptively change the permissions to #0777 to try and work around win32 permissions issues. - os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - os.remove(path) - - if os.path.exists(target_path): - if os.path.isdir(target_path): - shutil.rmtree(target_path, ignore_errors=True) - else: - os.remove(target_path) - - # shutil.move() command creates intermediate directories if they do not exist. - shutil.move(source_path, target_path) - - # ensure all the new directories are importable. - intermediate_dirs = os.path.dirname(os.path.relpath(target_path, self._target_dir)) - parent_dirname = self._target_dir - for dirname in intermediate_dirs.split(os.sep): - parent_dirname = os.path.join(parent_dirname, dirname) - self._make_package(parent_dirname) - - self._record_url_downloaded(package_name, url) - - def install(self, url, should_refresh=False, target_name=None, - url_subpath=None, files_to_remove=None): - """Install a python package from an URL. - - Args: - url: The URL from which to download the package. - - Optional Args: - should_refresh: A boolean value of whether the package should be - downloaded again if the package is already present. - target_name: The name of the folder or file in the autoinstaller - target directory at which the package should be - installed. Defaults to the base name of the - URL sub-path. This parameter must be provided if - the URL sub-path is not specified. - url_subpath: The relative path of the URL directory that should - be installed. Defaults to the full directory, or - the entire URL contents. - - """ - if target_name is None: - if not url_subpath: - raise ValueError('The "target_name" parameter must be ' - 'provided if the "url_subpath" parameter ' - "is not provided.") - # Remove any trailing slashes. - url_subpath = os.path.normpath(url_subpath) - target_name = os.path.basename(url_subpath) - - target_path = os.path.join(self._target_dir, target_name.replace('/', os.sep)) - if not should_refresh and self._is_downloaded(target_name, url): - return False - - files_to_remove = files_to_remove or [] - package_name = target_name.replace(os.sep, '.') - _log.info("Auto-installing package: %s" % package_name) - - # The scratch directory is where we will download and prepare - # files specific to this install until they are ready to move - # into place. - scratch_dir = self._create_scratch_directory(target_name) - - try: - self._install(package_name=package_name, - target_path=target_path, - scratch_dir=scratch_dir, - url=url, - url_subpath=url_subpath, - files_to_remove=files_to_remove) - except Exception, err: - # Append existing Error message to new Error. - message = ("Error auto-installing the %s package to:\n" - ' "%s"\n' - " --> Inner message: %s" - % (target_name, target_path, err)) - raise Exception(message) - finally: - shutil.rmtree(scratch_dir, ignore_errors=True) - _log.debug('Auto-installed %s to:' % url) - _log.debug(' "%s"' % target_path) - return True |