From c02733cb4859d82653c7d91a5bde090ce475d493 Mon Sep 17 00:00:00 2001 From: Richard Hughes Date: Wed, 10 Sep 2014 09:27:37 +0100 Subject: Move the distro helper scripts to another repo We don't want to clutter the git history with details of my RAID setup... --- contrib/helpers/fedora-20-gnome-3-12.sh | 19 -- contrib/helpers/fedora-20.sh | 22 -- contrib/helpers/fedora-21.sh | 29 --- contrib/helpers/fedora-rawhide.sh | 22 -- contrib/helpers/ubuntu-utopic.sh | 10 - contrib/packages/fedora-20/project.conf | 7 - contrib/packages/fedora-21/project.conf | 7 - contrib/packages/fedora-21/upload-metadata.sh | 3 - contrib/packages/fedora-21/upload-screenshots.sh | 3 - contrib/packages/fedora-download-cache.py | 260 --------------------- contrib/packages/fedora-rawhide/project.conf | 7 - .../screenshots/fedora-21/upload-screenshots.sh | 12 - 12 files changed, 401 deletions(-) delete mode 100755 contrib/helpers/fedora-20-gnome-3-12.sh delete mode 100755 contrib/helpers/fedora-20.sh delete mode 100755 contrib/helpers/fedora-21.sh delete mode 100755 contrib/helpers/fedora-rawhide.sh delete mode 100755 contrib/helpers/ubuntu-utopic.sh delete mode 100644 contrib/packages/fedora-20/project.conf delete mode 100644 contrib/packages/fedora-21/project.conf delete mode 100755 contrib/packages/fedora-21/upload-metadata.sh delete mode 100755 contrib/packages/fedora-21/upload-screenshots.sh delete mode 100755 contrib/packages/fedora-download-cache.py delete mode 100644 contrib/packages/fedora-rawhide/project.conf delete mode 100755 contrib/screenshots/fedora-21/upload-screenshots.sh (limited to 'contrib') diff --git a/contrib/helpers/fedora-20-gnome-3-12.sh b/contrib/helpers/fedora-20-gnome-3-12.sh deleted file mode 100755 index 3c3ccd0..0000000 --- a/contrib/helpers/fedora-20-gnome-3-12.sh +++ /dev/null @@ -1,19 +0,0 @@ -time client/appstream-builder \ - --api-version=0.41 \ - --log-dir=./contrib/logs/fedora-20-gnome-3-12 \ - --temp-dir=./tmp/fedora-20-gnome-3-12 \ - --cache-dir=./contrib/cache \ - --packages-dir=./contrib/packages/fedora-20-gnome-3-12/packages/ \ - --extra-appstream-dir=../fedora-appstream/appstream-extra \ - --extra-appdata-dir=../fedora-appstream/appdata-extra \ - --extra-screenshots-dir=../fedora-appstream/screenshots-extra \ - --output-dir=./contrib/metadata \ - --screenshot-dir=./contrib/screenshots/fedora-20-gnome-3-12 \ - --basename=fedora-20-gnome-3-12 \ - --screenshot-uri=http://alt.fedoraproject.org/pub/alt/screenshots/f20/ -./client/appstream-util status-html \ - ./contrib/metadata/fedora-20-gnome-3-12.xml.gz \ - ./contrib/screenshots/fedora-20-gnome-3-12/status.html -./client/appstream-util status-html \ - ./contrib/metadata/fedora-20-gnome-3-12-failed.xml.gz \ - ./contrib/screenshots/fedora-20-gnome-3-12/failed.html diff --git a/contrib/helpers/fedora-20.sh b/contrib/helpers/fedora-20.sh deleted file mode 100755 index 2c52e21..0000000 --- a/contrib/helpers/fedora-20.sh +++ /dev/null @@ -1,22 +0,0 @@ -time client/appstream-builder \ - --api-version=0.3 \ - --log-dir=./contrib/logs/fedora-20 \ - --temp-dir=./tmp/fedora-20 \ - --cache-dir=./contrib/cache \ - --packages-dir=./contrib/packages/fedora-20/packages/ \ - --extra-appstream-dir=../fedora-appstream/appstream-extra \ - --extra-appdata-dir=../fedora-appstream/appdata-extra \ - --extra-screenshots-dir=../fedora-appstream/screenshots-extra \ - --output-dir=./contrib/metadata \ - --screenshot-dir=./contrib/screenshots/fedora-20 \ - --basename=fedora-20 \ - --screenshot-uri=http://alt.fedoraproject.org/pub/alt/screenshots/f20/ -./client/appstream-util non-package-yaml \ - ./contrib/metadata/fedora-20.xml.gz \ - ./contrib/screenshots/fedora-20/applications-to-import.yaml -./client/appstream-util status-html \ - ./contrib/metadata/fedora-20.xml.gz \ - ./contrib/screenshots/fedora-20/status.html -./client/appstream-util status-html \ - ./contrib/metadata/fedora-20-failed.xml.gz \ - ./contrib/screenshots/fedora-20/failed.html diff --git a/contrib/helpers/fedora-21.sh b/contrib/helpers/fedora-21.sh deleted file mode 100755 index dd5bf0b..0000000 --- a/contrib/helpers/fedora-21.sh +++ /dev/null @@ -1,29 +0,0 @@ -time client/appstream-builder \ - --api-version=0.8 \ - --add-cache-id \ - --old-metadata=./contrib/screenshots/fedora-21 \ - --log-dir=../createrepo_as_logs \ - --temp-dir=./tmp/fedora-21 \ - --cache-dir=./contrib/cache \ - --packages-dir=./contrib/packages/fedora-21/packages/ \ - --extra-appstream-dir=../fedora-appstream/appstream-extra \ - --extra-appdata-dir=../fedora-appstream/appdata-extra \ - --extra-screenshots-dir=../fedora-appstream/screenshots-extra \ - --output-dir=./contrib/screenshots/fedora-21 \ - --screenshot-dir=./contrib/screenshots/fedora-21 \ - --basename=fedora-21 \ - --screenshot-uri=http://alt.fedoraproject.org/pub/alt/screenshots/f21/ -./client/appstream-util non-package-yaml \ - ./contrib/screenshots/fedora-21/fedora-21.xml.gz \ - ./contrib/screenshots/fedora-21/applications-to-import.yaml -./client/appstream-util status-html \ - ./contrib/screenshots/fedora-21/fedora-21.xml.gz \ - ./contrib/screenshots/fedora-21/status.html -./client/appstream-util status-html \ - ./contrib/screenshots/fedora-21/fedora-21-failed.xml.gz \ - ./contrib/screenshots/fedora-21/failed.html - -# sync the screenshots and metadata -cd contrib/screenshots/fedora-21/ -./upload-screenshots.sh -cd - diff --git a/contrib/helpers/fedora-rawhide.sh b/contrib/helpers/fedora-rawhide.sh deleted file mode 100755 index 59a728a..0000000 --- a/contrib/helpers/fedora-rawhide.sh +++ /dev/null @@ -1,22 +0,0 @@ -time client/appstream-builder \ - --api-version=0.8 \ - --log-dir=./contrib/logs/fedora-22 \ - --temp-dir=./tmp/fedora-22 \ - --cache-dir=./contrib/cache \ - --packages-dir=./contrib/packages/fedora-rawhide/packages/ \ - --extra-appstream-dir=../fedora-appstream/appstream-extra \ - --extra-appdata-dir=../fedora-appstream/appdata-extra \ - --extra-screenshots-dir=../fedora-appstream/screenshots-extra \ - --output-dir=./contrib/metadata \ - --screenshot-dir=./contrib/screenshots/fedora-22 \ - --basename=fedora-22 \ - --screenshot-uri=http://alt.fedoraproject.org/pub/alt/screenshots/f22/ -./client/appstream-util non-package-yaml \ - ./contrib/metadata/fedora-22.xml.gz \ - ./contrib/screenshots/fedora-22/applications-to-import.yaml -./client/appstream-util status-html \ - ./contrib/metadata/fedora-22.xml.gz \ - ./contrib/screenshots/fedora-22/status.html -./client/appstream-util status-html \ - ./contrib/metadata/fedora-22-failed.xml.gz \ - ./contrib/screenshots/fedora-22/failed.html diff --git a/contrib/helpers/ubuntu-utopic.sh b/contrib/helpers/ubuntu-utopic.sh deleted file mode 100755 index fe359b8..0000000 --- a/contrib/helpers/ubuntu-utopic.sh +++ /dev/null @@ -1,10 +0,0 @@ -time client/appstream-builder \ - --api-version=0.7 \ - --log-dir=./contrib/logs/ubuntu-utopic \ - --temp-dir=./contrib/tmp-ubu \ - --cache-dir=./contrib/cache-ubu \ - --packages-dir=/media/raid/Archive/Ubuntu/ \ - --extra-appdata-dir=../fedora-appstream/appdata-extra \ - --output-dir=./contrib/metadata \ - --screenshot-dir=./contrib/screenshots/ubuntu-utopic \ - --basename=ubuntu-utopic diff --git a/contrib/packages/fedora-20/project.conf b/contrib/packages/fedora-20/project.conf deleted file mode 100644 index 3e3acdd..0000000 --- a/contrib/packages/fedora-20/project.conf +++ /dev/null @@ -1,7 +0,0 @@ -[AppstreamProject] - -# the distro release tag -DistroTag=f20 - -# the repo ids used for this project -RepoIds=fedora,updates,updates-testing diff --git a/contrib/packages/fedora-21/project.conf b/contrib/packages/fedora-21/project.conf deleted file mode 100644 index 27b6c06..0000000 --- a/contrib/packages/fedora-21/project.conf +++ /dev/null @@ -1,7 +0,0 @@ -[AppstreamProject] - -# the distro release tag -DistroTag=f21 - -# the repo ids used for this project -RepoIds=fedora,updates,updates-testing diff --git a/contrib/packages/fedora-21/upload-metadata.sh b/contrib/packages/fedora-21/upload-metadata.sh deleted file mode 100755 index 36550ca..0000000 --- a/contrib/packages/fedora-21/upload-metadata.sh +++ /dev/null @@ -1,3 +0,0 @@ -scp fedora-*-icons.tar.gz hughsient@people.freedesktop.org:public_html/temp/ -scp fedora-*.xml.gz hughsient@people.freedesktop.org:public_html/temp/ -cp fedora-*-icons.tar.gz fedora-*.xml.gz $HOME/rpmbuild/SOURCES/ diff --git a/contrib/packages/fedora-21/upload-screenshots.sh b/contrib/packages/fedora-21/upload-screenshots.sh deleted file mode 100755 index fbfdecd..0000000 --- a/contrib/packages/fedora-21/upload-screenshots.sh +++ /dev/null @@ -1,3 +0,0 @@ -chmod a+rx screenshots/*x* -chmod a+r screenshots/*x*/*.png -rsync -v --progress ./screenshots/ rhughes@secondary01.fedoraproject.org:/srv/pub/alt/screenshots/f21/ diff --git a/contrib/packages/fedora-download-cache.py b/contrib/packages/fedora-download-cache.py deleted file mode 100755 index 7f521a7..0000000 --- a/contrib/packages/fedora-download-cache.py +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Licensed under the GNU General Public License Version 2 -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -# Copyright (C) 2009-2014 -# Richard Hughes -# - -import glob -import os -import rpm -import rpmUtils -import sys -import yum -import hashlib -import fnmatch -import datetime -import ConfigParser - -timestamp = datetime.datetime.now().strftime('%Y%m%d') - -_ts = rpm.ts() -_ts.setVSFlags(0x7FFFFFFF) - -class Config: - - def __init__(self): - - # get the project defaults - self.cfg_project = ConfigParser.ConfigParser() - self.cfg_project.read('./project.conf') - self.distro_tag = self.cfg_project.get('AppstreamProject', 'DistroTag') - self.repo_ids = self.cfg_project.get('AppstreamProject', 'RepoIds').split(',') - -def _do_newest_filtering(pkglist): - ''' - Only return the newest package for each name.arch - ''' - newest = {} - for pkg in pkglist: - key = (pkg.name, pkg.arch) - if key in newest: - - # the current package is older - if pkg.verCMP(newest[key]) < 0: - continue - - # the current package is the same version - if pkg.verCMP(newest[key]) == 0: - continue - - # the current package is newer than what we have stored - del newest[key] - - newest[key] = pkg - return newest.values() - -def search_package_list(pkg): - for instfile in pkg.returnFileEntries(): - if fnmatch.fnmatch(instfile, '/usr/share/applications/*.desktop'): - return True - if fnmatch.fnmatch(instfile, '/usr/share/applications/kde4/*.desktop'): - return True - if fnmatch.fnmatch(instfile, '/usr/share/fonts/*/*.otf'): - return True - if fnmatch.fnmatch(instfile, '/usr/share/fonts/*/*.ttf'): - return True - if fnmatch.fnmatch(instfile, '/usr/share/ibus/component/*.xml'): - return True - if fnmatch.fnmatch(instfile, '/usr/share/ibus-table/tables/*.db'): - return True - if fnmatch.fnmatch(instfile, '/usr/lib64/gstreamer-1.0/libgst*.so'): - return True - if fnmatch.fnmatch(instfile, '/usr/share/appdata/*.metainfo.xml'): - return True - return False - -def get_sha256_hash(filename, block_size=256*128): - md5 = hashlib.sha256() - with open(filename,'rb') as f: - for chunk in iter(lambda: f.read(block_size), b''): - md5.update(chunk) - return md5.hexdigest() - -def ensure_pkg_exists(yb, existing, pkg): - - # get base name without the slash - relativepath = pkg.returnSimple('relativepath') - pos = relativepath.rfind('/') - if pos != -1: - relativepath = relativepath[pos+1:] - - # is in cache? - path = './packages/' + relativepath - if os.path.exists(path) and get_sha256_hash(path) == pkg.checksum: - #print("INFO: %s up to date" % pkg.nvra) - return - - # do we have an old version of this? - if existing.has_key(pkg.name) and os.path.exists(existing[pkg.name]): - print("INFO: deleting %s" % os.path.basename(existing[pkg.name])) - os.remove(existing[pkg.name]) - - # make sure the metadata exists - repo = yb.repos.getRepo(pkg.repoid) - - # download now - print("INFO: downloading %s" % os.path.basename(path)) - pkg.localpath = path - repo.getPackage(pkg) - -def pkg_from_name(sack, name): - for pkg in sack: - if pkg.name == name: - return pkg - return None - -def update(): - - # create if we're starting from nothing - if not os.path.exists('./packages'): - os.makedirs('./packages') - - # get extra packages needed for some applications - cfg = Config() - extra_packages = [] - extra_packages.append('alliance') - extra_packages.append('hotot-common') - extra_packages.append('java-1.7.0-openjdk') - extra_packages.append('kchmviewer') - extra_packages.append('libprojectM-qt') - extra_packages.append('nntpgrab-core') - extra_packages.append('scummvm') - extra_packages.append('switchdesk') - extra_packages.append('transmission-common') - extra_packages.append('oxygen-icon-theme') - - # find out what we've got already - files = glob.glob("./packages/*.rpm") - files.sort() - existing = {} - for f in files: - fd = os.open(f, os.O_RDONLY) - try: - hdr = _ts.hdrFromFdno(fd) - except Exception as e: - pass - else: - existing[hdr.name] = f - os.close(fd) - print("INFO: Found %i existing packages for %s" % (len(existing), cfg.distro_tag)) - - # setup yum - yb = yum.YumBase() - yb.preconf.releasever = cfg.distro_tag[1:] - yb.doConfigSetup(errorlevel=-1, debuglevel=-1) - yb.conf.cache = 0 - - # what is native for this arch - basearch = rpmUtils.arch.getBaseArch() - if basearch == 'i386': - basearch_list = ['i386', 'i486', 'i586', 'i686'] - else: - basearch_list = [basearch] - basearch_list.append('noarch') - - # reget the correct metadata every day - yb.repos.enableRepo('rawhide') - for repo in yb.repos.listEnabled(): - if repo.id in cfg.repo_ids: - repo.enable() - repo.metadata_expire = 60 * 60 * 12 # 12 hours - print("INFO: enabled: %s" % repo.id) - else: - repo.disable() - print("INFO: disabled: %s" % repo.id) - - # find all packages - print("INFO: Checking metadata...") - downloaded = {} - try: - pkgs = yb.pkgSack - except yum.Errors.NoMoreMirrorsRepoError as e: - print("FAILED:" % str(e)) - sys.exit(1) - newest_packages = _do_newest_filtering(pkgs) - - # restrict this down so we have less to search - print("INFO: Finding interesting packages...") - available_packages = [] - matched_packages = [] - for pkg in newest_packages: - if pkg.repoid in cfg.repo_ids and pkg.arch in basearch_list: - available_packages.append(pkg) - if pkg.name in extra_packages or search_package_list(pkg): - matched_packages.append(pkg) - - # ensure the package, and the first level of deps is downloaded - print("INFO: Downloading packages...") - matched_packages.sort() - for pkg in matched_packages: - #print("INFO: Checking for %s..." % pkg.name) - ensure_pkg_exists(yb, existing, pkg) - for require in pkg.strong_requires_names: - # remove helpful package suffixes - idx = require.find('(x86-64)') - if idx > 0: - require = require[0:idx] - if downloaded.has_key(require): - continue - dep = pkg_from_name(available_packages, require) - if not dep: - continue - if dep.base_package_name != pkg.base_package_name and not require.startswith(pkg.name): - continue - #print ("INFO: " + pkg.name + " also needs " + require) - if require in extra_packages: - print("WARNING: Remove %s from whitelist" % require) - ensure_pkg_exists(yb, existing, dep) - downloaded[dep.name] = True - downloaded[pkg.name] = True - - if len(downloaded) == 0: - print("INFO: no packages downloaded for %s" % cfg.distro_tag) - return - - # have any packages been removed? - for i in existing: - if not downloaded.has_key(i): - print("INFO: deleting %s" % existing[i]) - os.remove(existing[i]) - -def main(): - - # check we're not top level - if os.path.exists('./application.py'): - print 'You cannot run these tools from the top level directory' - sys.exit(1) - - # update all the packages - update() - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/contrib/packages/fedora-rawhide/project.conf b/contrib/packages/fedora-rawhide/project.conf deleted file mode 100644 index 7d49524..0000000 --- a/contrib/packages/fedora-rawhide/project.conf +++ /dev/null @@ -1,7 +0,0 @@ -[AppstreamProject] - -# the distro release tag -DistroTag=f21 - -# the repo ids used for this project -RepoIds=rawhide diff --git a/contrib/screenshots/fedora-21/upload-screenshots.sh b/contrib/screenshots/fedora-21/upload-screenshots.sh deleted file mode 100755 index fd4ce58..0000000 --- a/contrib/screenshots/fedora-21/upload-screenshots.sh +++ /dev/null @@ -1,12 +0,0 @@ -chmod a+rx 112x63 -chmod a+rx 624x351 -chmod a+rx 752x423 -chmod a+rx source -chmod a+r */*.png -chmod a+r *.html -rsync -vr --progress \ - 112x63 624x351 752x423 \ - fedora-21.xml.gz fedora-21-icons.tar.gz \ - *.html \ - applications-to-import.yaml \ - rhughes@secondary01.fedoraproject.org:/srv/pub/alt/screenshots/f21/ -- cgit v1.2.1