summaryrefslogtreecommitdiff
path: root/Tools/Scripts/webkitpy/common/system/autoinstall.py
diff options
context:
space:
mode:
authorKonstantin Tokarev <annulen@yandex.ru>2016-08-25 19:20:41 +0300
committerKonstantin Tokarev <annulen@yandex.ru>2017-02-02 12:30:55 +0000
commit6882a04fb36642862b11efe514251d32070c3d65 (patch)
treeb7959826000b061fd5ccc7512035c7478742f7b0 /Tools/Scripts/webkitpy/common/system/autoinstall.py
parentab6df191029eeeb0b0f16f127d553265659f739e (diff)
downloadqtwebkit-6882a04fb36642862b11efe514251d32070c3d65.tar.gz
Imported QtWebKit TP3 (git b57bc6801f1876c3220d5a4bfea33d620d477443)
Change-Id: I3b1d8a2808782c9f34d50240000e20cb38d3680f Reviewed-by: Konstantin Tokarev <annulen@yandex.ru>
Diffstat (limited to 'Tools/Scripts/webkitpy/common/system/autoinstall.py')
-rw-r--r--Tools/Scripts/webkitpy/common/system/autoinstall.py433
1 files changed, 0 insertions, 433 deletions
diff --git a/Tools/Scripts/webkitpy/common/system/autoinstall.py b/Tools/Scripts/webkitpy/common/system/autoinstall.py
deleted file mode 100644
index 2e15887bb..000000000
--- a/Tools/Scripts/webkitpy/common/system/autoinstall.py
+++ /dev/null
@@ -1,433 +0,0 @@
-# Copyright (c) 2009, Daniel Krech All rights reserved.
-# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org)
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# * Neither the name of the Daniel Krech nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Support for automatically downloading Python packages from an URL."""
-
-
-import codecs
-import logging
-import os
-import shutil
-import stat
-import sys
-import tarfile
-import tempfile
-import urllib2
-import urlparse
-import zipfile
-
-_log = logging.getLogger(__name__)
-
-
-class AutoInstaller(object):
-
- """Supports automatically installing Python packages from an URL.
-
- Supports uncompressed files, .tar.gz, and .zip formats.
-
- Basic usage:
-
- installer = AutoInstaller()
-
- installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b",
- url_subpath="pep8-0.5.0/pep8.py")
- installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.4.zip",
- url_subpath="mechanize")
-
- """
-
- def __init__(self, append_to_search_path=False, make_package=True,
- target_dir=None, temp_dir=None):
- """Create an AutoInstaller instance, and set up the target directory.
-
- Args:
- append_to_search_path: A boolean value of whether to append the
- target directory to the sys.path search path.
- make_package: A boolean value of whether to make the target
- directory a package. This adds an __init__.py file
- to the target directory -- allowing packages and
- modules within the target directory to be imported
- explicitly using dotted module names.
- target_dir: The directory path to which packages should be installed.
- Defaults to a subdirectory of the folder containing
- this module called "autoinstalled".
- temp_dir: The directory path to use for any temporary files
- generated while downloading, unzipping, and extracting
- packages to install. Defaults to a standard temporary
- location generated by the tempfile module. This
- parameter should normally be used only for development
- testing.
-
- """
- if target_dir is None:
- this_dir = os.path.dirname(__file__)
- target_dir = os.path.join(this_dir, "autoinstalled")
-
- # Ensure that the target directory exists.
- self._set_up_target_dir(target_dir, append_to_search_path, make_package)
-
- self._target_dir = target_dir
- self._temp_dir = temp_dir
-
- def _write_file(self, path, text, encoding):
- with codecs.open(path, "w", encoding) as filehandle:
- filehandle.write(text)
-
- def _set_up_target_dir(self, target_dir, append_to_search_path,
- make_package):
- """Set up a target directory.
-
- Args:
- target_dir: The path to the target directory to set up.
- append_to_search_path: A boolean value of whether to append the
- target directory to the sys.path search path.
- make_package: A boolean value of whether to make the target
- directory a package. This adds an __init__.py file
- to the target directory -- allowing packages and
- modules within the target directory to be imported
- explicitly using dotted module names.
-
- """
- if not os.path.exists(target_dir):
- os.makedirs(target_dir)
-
- if append_to_search_path:
- sys.path.append(target_dir)
-
- if make_package:
- self._make_package(target_dir)
-
- def _make_package(self, target_dir):
- init_path = os.path.join(target_dir, "__init__.py")
- if not os.path.exists(init_path):
- text = ("# This file is required for Python to search this "
- "directory for modules.\n")
- self._write_file(init_path, text, "ascii")
-
- def _create_scratch_directory_inner(self, prefix):
- """Create a scratch directory without exception handling.
-
- Creates a scratch directory inside the AutoInstaller temp
- directory self._temp_dir, or inside a platform-dependent temp
- directory if self._temp_dir is None. Returns the path to the
- created scratch directory.
-
- Raises:
- OSError: [Errno 2] if the containing temp directory self._temp_dir
- is not None and does not exist.
-
- """
- # The tempfile.mkdtemp() method function requires that the
- # directory corresponding to the "dir" parameter already exist
- # if it is not None.
- scratch_dir = tempfile.mkdtemp(prefix=prefix.replace('/', '.'), dir=self._temp_dir)
- return scratch_dir
-
- def _create_scratch_directory(self, target_name):
- """Create a temporary scratch directory, and return its path.
-
- The scratch directory is generated inside the temp directory
- of this AutoInstaller instance. This method also creates the
- temp directory if it does not already exist.
-
- """
- prefix = target_name.replace(os.sep, "_") + "_"
- try:
- scratch_dir = self._create_scratch_directory_inner(prefix)
- except OSError:
- # Handle case of containing temp directory not existing--
- # OSError: [Errno 2] No such file or directory:...
- temp_dir = self._temp_dir
- if temp_dir is None or os.path.exists(temp_dir):
- raise
- # Else try again after creating the temp directory.
- os.makedirs(temp_dir)
- scratch_dir = self._create_scratch_directory_inner(prefix)
-
- return scratch_dir
-
- def _url_downloaded_path(self, target_name):
- return os.path.join(self._target_dir, ".%s.url" % target_name.replace('/', '_'))
-
- def _is_downloaded(self, target_name, url):
- version_path = self._url_downloaded_path(target_name)
-
- if not os.path.exists(version_path):
- return False
-
- with codecs.open(version_path, "r", "utf-8") as filehandle:
- return filehandle.read().strip() == url.strip()
-
- def _record_url_downloaded(self, target_name, url):
- version_path = self._url_downloaded_path(target_name)
- self._write_file(version_path, url, "utf-8")
-
- def _extract_targz(self, path, scratch_dir):
- # tarfile.extractall() extracts to a path without the trailing ".tar.gz".
- target_basename = os.path.basename(path[:-len(".tar.gz")])
- target_path = os.path.join(scratch_dir, target_basename)
-
- try:
- tar_file = tarfile.open(path)
- except tarfile.ReadError, err:
- # Append existing Error message to new Error.
- message = ("Could not open tar file: %s\n"
- " The file probably does not have the correct format.\n"
- " --> Inner message: %s"
- % (path, err))
- raise Exception(message)
-
- try:
- tar_file.extractall(target_path)
- finally:
- tar_file.close()
-
- return target_path
-
- # This is a replacement for ZipFile.extractall(), which is
- # available in Python 2.6 but not in earlier versions.
- # NOTE: The version in 2.6.1 (which shipped on Snow Leopard) is broken!
- def _extract_all(self, zip_file, target_dir):
- for name in zip_file.namelist():
- path = os.path.join(target_dir, name)
- if not os.path.basename(path):
- # Then the path ends in a slash, so it is a directory.
- os.makedirs(path)
- continue
-
- try:
- # We open this file w/o encoding, as we're reading/writing
- # the raw byte-stream from the zip file.
- outfile = open(path, 'wb')
- except IOError:
- # Not all zip files seem to list the directories explicitly,
- # so try again after creating the containing directory.
- _log.debug("Got IOError: retrying after creating directory...")
- dirname = os.path.dirname(path)
- os.makedirs(dirname)
- outfile = open(path, 'wb')
-
- try:
- outfile.write(zip_file.read(name))
- finally:
- outfile.close()
-
- def _unzip(self, path, scratch_dir):
- # zipfile.extractall() extracts to a path without the trailing ".zip".
- target_basename = os.path.basename(path[:-len(".zip")])
- target_path = os.path.join(scratch_dir, target_basename)
-
- try:
- zip_file = zipfile.ZipFile(path, "r")
- except zipfile.BadZipfile, err:
- message = ("Could not open zip file: %s\n"
- " --> Inner message: %s"
- % (path, err))
- raise Exception(message)
-
- try:
- self._extract_all(zip_file, scratch_dir)
- finally:
- zip_file.close()
-
- return target_path
-
- def _prepare_package(self, path, scratch_dir):
- """Prepare a package for use, if necessary, and return the new path.
-
- For example, this method unzips zipped files and extracts
- tar files.
-
- Args:
- path: The path to the downloaded URL contents.
- scratch_dir: The scratch directory. Note that the scratch
- directory contains the file designated by the
- path parameter.
-
- """
- # FIXME: Add other natural extensions.
- if path.endswith(".zip"):
- new_path = self._unzip(path, scratch_dir)
- elif path.endswith(".tar.gz"):
- new_path = self._extract_targz(path, scratch_dir)
- else:
- # No preparation is needed.
- new_path = path
-
- return new_path
-
- def _download_to_stream(self, url, stream):
- failures = 0
- while True:
- try:
- netstream = urllib2.urlopen(url)
- break
- except IOError, err:
- # Try multiple times
- if failures < 5:
- _log.warning("Failed to download %s, %s retrying" % (
- url, err))
- failures += 1
- continue
-
- # Append existing Error message to new Error.
- message = ('Could not download Python modules from URL "%s".\n'
- " Make sure you are connected to the internet.\n"
- " You must be connected to the internet when "
- "downloading needed modules for the first time.\n"
- " --> Inner message: %s"
- % (url, err))
- raise IOError(message)
- code = 200
- if hasattr(netstream, "getcode"):
- code = netstream.getcode()
- if not 200 <= code < 300:
- raise ValueError("HTTP Error code %s" % code)
-
- BUFSIZE = 2**13 # 8KB
- while True:
- data = netstream.read(BUFSIZE)
- if not data:
- break
- stream.write(data)
- netstream.close()
-
- def _download(self, url, scratch_dir):
- url_path = urlparse.urlsplit(url)[2]
- url_path = os.path.normpath(url_path) # Removes trailing slash.
- target_filename = os.path.basename(url_path)
- target_path = os.path.join(scratch_dir, target_filename)
-
- with open(target_path, "wb") as stream:
- self._download_to_stream(url, stream)
-
- return target_path
-
- def _install(self, scratch_dir, package_name, target_path, url, url_subpath, files_to_remove):
- """Install a python package from an URL.
-
- This internal method overwrites the target path if the target
- path already exists.
-
- """
- path = self._download(url=url, scratch_dir=scratch_dir)
- path = self._prepare_package(path, scratch_dir)
-
- if url_subpath is None:
- source_path = path
- else:
- source_path = os.path.join(path, url_subpath)
-
- for filename in files_to_remove:
- path = os.path.join(source_path, filename.replace('/', os.sep))
- if os.path.exists(path):
- # Pre-emptively change the permissions to #0777 to try and work around win32 permissions issues.
- os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
- os.remove(path)
-
- if os.path.exists(target_path):
- if os.path.isdir(target_path):
- shutil.rmtree(target_path, ignore_errors=True)
- else:
- os.remove(target_path)
-
- # shutil.move() command creates intermediate directories if they do not exist.
- shutil.move(source_path, target_path)
-
- # ensure all the new directories are importable.
- intermediate_dirs = os.path.dirname(os.path.relpath(target_path, self._target_dir))
- parent_dirname = self._target_dir
- for dirname in intermediate_dirs.split(os.sep):
- parent_dirname = os.path.join(parent_dirname, dirname)
- self._make_package(parent_dirname)
-
- self._record_url_downloaded(package_name, url)
-
- def install(self, url, should_refresh=False, target_name=None,
- url_subpath=None, files_to_remove=None):
- """Install a python package from an URL.
-
- Args:
- url: The URL from which to download the package.
-
- Optional Args:
- should_refresh: A boolean value of whether the package should be
- downloaded again if the package is already present.
- target_name: The name of the folder or file in the autoinstaller
- target directory at which the package should be
- installed. Defaults to the base name of the
- URL sub-path. This parameter must be provided if
- the URL sub-path is not specified.
- url_subpath: The relative path of the URL directory that should
- be installed. Defaults to the full directory, or
- the entire URL contents.
-
- """
- if target_name is None:
- if not url_subpath:
- raise ValueError('The "target_name" parameter must be '
- 'provided if the "url_subpath" parameter '
- "is not provided.")
- # Remove any trailing slashes.
- url_subpath = os.path.normpath(url_subpath)
- target_name = os.path.basename(url_subpath)
-
- target_path = os.path.join(self._target_dir, target_name.replace('/', os.sep))
- if not should_refresh and self._is_downloaded(target_name, url):
- return False
-
- files_to_remove = files_to_remove or []
- package_name = target_name.replace(os.sep, '.')
- _log.info("Auto-installing package: %s" % package_name)
-
- # The scratch directory is where we will download and prepare
- # files specific to this install until they are ready to move
- # into place.
- scratch_dir = self._create_scratch_directory(target_name)
-
- try:
- self._install(package_name=package_name,
- target_path=target_path,
- scratch_dir=scratch_dir,
- url=url,
- url_subpath=url_subpath,
- files_to_remove=files_to_remove)
- except Exception, err:
- # Append existing Error message to new Error.
- message = ("Error auto-installing the %s package to:\n"
- ' "%s"\n'
- " --> Inner message: %s"
- % (target_name, target_path, err))
- raise Exception(message)
- finally:
- shutil.rmtree(scratch_dir, ignore_errors=True)
- _log.debug('Auto-installed %s to:' % url)
- _log.debug(' "%s"' % target_path)
- return True