Hi, Here is a hotfix for fedimg. Right now Rawhide messages are not getting processed. This hotfix will fix the issue. PR for the same[1] [1] https://github.com/fedora-infra/fedimg/pull/103 +1s >From b52a1dd1eda25bfbade026a1ebde93839fa3379b Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> Date: Thu, 19 Apr 2018 22:05:27 +0530 Subject: [PATCH 1/2] fedimg: Add the intial files for PR#103 Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> --- files/hotfix/fedimg/consumers.py | 160 +++++++++++++++++++++++++++++++++++++++ roles/fedimg/tasks/main.yml | 8 ++ 2 files changed, 168 insertions(+) create mode 100644 files/hotfix/fedimg/consumers.py diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py new file mode 100644 index 0000000..ce4d662 --- /dev/null +++ b/files/hotfix/fedimg/consumers.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# This file is part of fedimg. +# Copyright (C) 2014-2017 Red Hat, Inc. +# +# fedimg is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# fedimg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with fedimg; if not, see http://www.gnu.org/licenses, +# or write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: David Gay <dgay@xxxxxxxxxx> +# Sayan Chowdhury <sayanchowdhury@xxxxxxxxxxxxxxxxx> +""" +This is the `fedmsg consumer`_ that subscribes to the topic emitted after the +completion of the nightly and production compose. The consumer on receving the +message uploads the image using the API of the cloud providers. +""" + +import logging +import multiprocessing.pool + +import fedmsg.consumers +import fedmsg.encoding +import fedfind.release +import fedfind.exceptions + +import fedimg.uploader + +from fedimg.config import PROCESS_COUNT, STATUS_FILTER +from fedimg.utils import get_rawxz_urls, get_value_from_dict + +_log = logging.getLogger(__name__) + + +class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): + """ + A `fedmsg consumer`_ that listens to the pungi compose topics and kicks + of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.prod.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.prod.enabled``. + """ + topic = ['org.fedoraproject.prod.pungi.compose.status.change'] + config_key = "fedimgconsumer.prod.enabled" + + def __init__(self, *args, **kwargs): + _log.info("FedimgConsumer initializing") + super(FedimgConsumer, self).__init__(*args, **kwargs) + + # Threadpool for upload jobs + _log.info("Creating thread pool of %s process", PROCESS_COUNT) + self.upload_pool = multiprocessing.pool.ThreadPool( + processes=PROCESS_COUNT + ) + _log.info("FedimgConsumer initialized") + + def consume(self, msg): + """ + This is called when we receive a message matching our topics. + + Args: + msg (dict): The raw message from fedmsg. + """ + _log.info('Received %r %r', msg['topic'], msg['body']['msg_id']) + + msg_info = msg['body']['msg'] + if msg_info['status'] not in STATUS_FILTER: + _log.debug('%s is not valid status' % msg_info['status']) + return + + location = msg_info['location'] + compose_id = msg_info['compose_id'] + try: + compose_metadata = fedfind.release.get_release(cid=compose_id).metadata + except fedfind.exceptions.UnsupportedComposeError: + LOG.debug("%r is unsupported compose" % compose_id) + return + + + # Till F27, both cloud-base and atomic images were available + # under variant CloudImages. With F28 and onward releases, + # cloud-base image compose moved to cloud variant and atomic images + # moved under atomic variant. + prev_rel = ['26', '27'] + if msg_info['release_version'] in prev_rel: + images_meta = get_value_from_dict( + compose_metadata, 'images', 'payload', 'images', 'CloudImages', + 'x86_64') + else: + images_meta = get_value_from_dict( + compose_metadata, 'images', 'payload', 'images', + 'Cloud', 'x86_64') + images_meta.extend(get_value_from_dict( + compose_metadata, 'images', 'payload', + 'images', 'AtomicHost', 'x86_64')) + + images_meta = get_value_from_dict( + compose_metadata, + 'images', + 'payload', + 'images', + 'CloudImages', + 'x86_64' + ) + + if images_meta is None: + _log.debug('No compatible image found to process') + return + + upload_urls = get_rawxz_urls(location, images_meta) + if len(upload_urls) > 0: + _log.info("Start processing compose id: %s", compose_id) + fedimg.uploader.upload( + pool=self.upload_pool, + urls=upload_urls, + compose_id=compose_id, + push_notifications=True + ) + + +class FedimgStagingConsumer(FedimgConsumer): + """ + A `fedmsg consumer`_ that listens to the staging pungi compose topics and + kicks of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.stg.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.stg.enabled``. + """ + topic = ['org.fedoraproject.stg.pungi.compose.status.change'] + config_key = "fedimgconsumer.stg.enabled" + + +class FedimgDevConsumer(FedimgConsumer): + """ + A `fedmsg consumer`_ that listens to the dev pungi compose topics and + kicks of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.dev.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.dev.enabled``. + """ + topic = ['org.fedoraproject.dev.pungi.compose.status.change'] + config_key = "fedimgconsumer.dev.enabled" diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index b0f18bf..2e35608 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -130,3 +130,11 @@ tags: - fedimg - hotfix + +- name: hotfix - copy the consumers.py over to the site-packages + copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py + notify: + - restart fedmsg-hub + tags: + - fedimg + - hotfix -- 2.9.4 >From 1bf3db45c8154d0a6a0f29836ec14ce24fa404c1 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> Date: Thu, 19 Apr 2018 22:07:12 +0530 Subject: [PATCH 2/2] fedimg: Add the patch for the PR#103, fix processing Rawhide messages Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> --- files/hotfix/fedimg/consumers.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py index ce4d662..8195321 100644 --- a/files/hotfix/fedimg/consumers.py +++ b/files/hotfix/fedimg/consumers.py @@ -85,7 +85,7 @@ class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): try: compose_metadata = fedfind.release.get_release(cid=compose_id).metadata except fedfind.exceptions.UnsupportedComposeError: - LOG.debug("%r is unsupported compose" % compose_id) + _log.debug("%r is unsupported compose" % compose_id) return @@ -106,15 +106,6 @@ class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): compose_metadata, 'images', 'payload', 'images', 'AtomicHost', 'x86_64')) - images_meta = get_value_from_dict( - compose_metadata, - 'images', - 'payload', - 'images', - 'CloudImages', - 'x86_64' - ) - if images_meta is None: _log.debug('No compatible image found to process') return -- 2.9.4 -- Sayan Chowdhury <https://sayanchowdhury.dgplug.org/> Senior Software Engineer, Fedora Engineering - Emerging Platform GPG Fingerprint : 0F16 E841 E517 225C 7D13 AB3C B023 9931 9CD0 5C8B Proud to work at The Open Organization! _______________________________________________ infrastructure mailing list -- infrastructure@xxxxxxxxxxxxxxxxxxxxxxx To unsubscribe send an email to infrastructure-leave@xxxxxxxxxxxxxxxxxxxxxxx
