This looks workable +1 On 26 March 2018 at 09:30, Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> wrote: > Hi, > > This is similar to the fedimg patch sent to the list earlier. > Autocloud right now is not processing the messages because of the > change in the F28 messages. This patch would fix the issue. > > This also contains a commit to remove a earlier hotfix in autocloud. > That hotfix was deployed during the last release but I forgot to > remove the hotfix. Don't know if this is proper but if needed I can > break this into two FBRs too. > > Here is the PR to the corresponding change[0] > > Thoughts? +1s? > > [0] https://github.com/kushaldas/autocloud/pull/62 > > From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> > Date: Mon, 26 Mar 2018 18:42:33 +0530 > Subject: [PATCH 1/3] autocloud, hotfix: Remove the autocloud hotfix for the > PR#56 > > Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> > --- > files/hotfix/autocloud/consumer.py | 135 --------------------------------- > roles/autocloud/backend/tasks/main.yml | 14 ---- > 2 files changed, 149 deletions(-) > delete mode 100644 files/hotfix/autocloud/consumer.py > > diff --git a/files/hotfix/autocloud/consumer.py > b/files/hotfix/autocloud/consumer.py > deleted file mode 100644 > index c216553..0000000 > --- a/files/hotfix/autocloud/consumer.py > +++ /dev/null > @@ -1,135 +0,0 @@ > -# -*- coding: utf-8 -*- > -from datetime import datetime > - > -import requests > -import fedmsg.consumers > -import fedfind.release > - > -from sqlalchemy import exc > - > -import autocloud > - > -from autocloud.models import init_model, ComposeDetails, ComposeJobDetails > -from autocloud.producer import publish_to_fedmsg > -from autocloud.utils import is_valid_image, produce_jobs > - > -import logging > -log = logging.getLogger("fedmsg") > - > -DEBUG = autocloud.DEBUG > - > - > -class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): > - """ > - Fedmsg consumer for Autocloud > - """ > - > - if DEBUG: > - topic = [ > - 'org.fedoraproject.dev.__main__.pungi.compose.status.change' > - ] > - > - else: > - topic = [ > - 'org.fedoraproject.prod.pungi.compose.status.change' > - ] > - > - config_key = 'autocloud.consumer.enabled' > - > - def __init__(self, *args, **kwargs): > - self.supported_archs = [arch for arch, _ in > ComposeJobDetails.ARCH_TYPES] > - > - log.info("Autocloud Consumer is ready for action.") > - super(AutoCloudConsumer, self).__init__(*args, **kwargs) > - > - def consume(self, msg): > - """ This is called when we receive a message matching the topic. """ > - > - log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) > - > - STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) > - VARIANTS_F = ('CloudImages',) > - > - images = [] > - compose_db_update = False > - msg_body = msg['body'] > - status = msg_body['msg']['status'] > - compose_images_json = None > - > - if status in STATUS_F: > - location = msg_body['msg']['location'] > - json_metadata = '{}/metadata/images.json'.format(location) > - resp = requests.get(json_metadata) > - compose_images_json = getattr(resp, 'json', False) > - > - if compose_images_json is not None: > - compose_images_json = compose_images_json() > - compose_images = compose_images_json['payload']['images'] > - compose_details = compose_images_json['payload']['compose'] > - compose_images = dict((variant, compose_images[variant]) > - for variant in VARIANTS_F > - if variant in compose_images) > - compose_id = compose_details['id'] > - rel = fedfind.release.get_release(cid=compose_id) > - release = rel.release > - compose_details.update({'release': release}) > - > - compose_images_variants = [variant for variant in VARIANTS_F > - if variant in compose_images] > - > - for variant in compose_images_variants: > - compose_image = compose_images[variant] > - for arch, payload in compose_image.iteritems(): > - > - if arch not in self.supported_archs: > - continue > - > - for item in payload: > - relative_path = item['path'] > - if not is_valid_image(relative_path): > - continue > - absolute_path = '{}/{}'.format(location, relative_path) > - item.update({ > - 'compose': compose_details, > - 'absolute_path': absolute_path, > - }) > - images.append(item) > - compose_db_update = True > - > - if compose_db_update: > - session = init_model() > - compose_date = > datetime.strptime(compose_details['date'], '%Y%m%d') > - try: > - cd = ComposeDetails( > - date=compose_date, > - compose_id=compose_details['id'], > - respin=compose_details['respin'], > - type=compose_details['type'], > - status=u'q', > - location=location, > - ) > - > - session.add(cd) > - session.commit() > - > - compose_details.update({ > - 'status': 'queued', > - 'compose_job_id': cd.id, > - }) > - publish_to_fedmsg(topic='compose.queued', > - **compose_details) > - except exc.IntegrityError: > - session.rollback() > - cd = session.query(ComposeDetails).filter_by( > - compose_id=compose_details['id']).first() > - log.info('Compose already exists %s: %s' % ( > - compose_details['id'], > - cd.id > - )) > - session.close() > - > - num_images = len(images) > - for pos, image in enumerate(images): > - image.update({'pos': (pos+1, num_images)}) > - > - produce_jobs(images) > diff --git a/roles/autocloud/backend/tasks/main.yml > b/roles/autocloud/backend/tasks/main.yml > index bc50df1..7ff5811 100644 > --- a/roles/autocloud/backend/tasks/main.yml > +++ b/roles/autocloud/backend/tasks/main.yml > @@ -139,17 +139,3 @@ > tags: > - autocloud > - autocloud/backend > - > -# > -# Install hotfix to ignore new architectures > -# See PR - https://github.com/kushaldas/autocloud/pull/56/ > -# > -- name: hotfix - copy over consumer files > - copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }} > - with_items: > - - { src: 'hotfix/autocloud/consumer.py', dest: > '/usr/lib/python2.7/site-packages/autocloud' } > - notify: > - - restart fedmsg-hub > - tags: > - - autocloud > - - hotfix > -- > 2.9.4 > > > From 7593e3dddaf85011b27c989a3f14c9794f4265bc Mon Sep 17 00:00:00 2001 > From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> > Date: Mon, 26 Mar 2018 18:50:42 +0530 > Subject: [PATCH 2/3] autocloud, hotfix: Add the hotfix to process F28 Atomic, > Cloud messages > > Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> > --- > files/hotfix/autocloud/consumer.py | 136 +++++++++++++++++++++++++++++++++ > roles/autocloud/backend/tasks/main.yml | 14 ++++ > 2 files changed, 150 insertions(+) > create mode 100644 files/hotfix/autocloud/consumer.py > > diff --git a/files/hotfix/autocloud/consumer.py > b/files/hotfix/autocloud/consumer.py > new file mode 100644 > index 0000000..a7b2aa5 > --- /dev/null > +++ b/files/hotfix/autocloud/consumer.py > @@ -0,0 +1,136 @@ > +# -*- coding: utf-8 -*- > +from datetime import datetime > + > +import requests > +import fedmsg.consumers > +import fedfind.release > + > +from sqlalchemy import exc > + > +import autocloud > + > +from autocloud.models import init_model, ComposeDetails, ComposeJobDetails > +from autocloud.producer import publish_to_fedmsg > +from autocloud.utils import is_valid_image, produce_jobs > + > +import logging > +log = logging.getLogger("fedmsg") > + > +DEBUG = autocloud.DEBUG > + > + > +class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): > + """ > + Fedmsg consumer for Autocloud > + """ > + > + if DEBUG: > + topic = [ > + 'org.fedoraproject.dev.__main__.pungi.compose.status.change' > + ] > + > + else: > + topic = [ > + 'org.fedoraproject.prod.pungi.compose.status.change' > + ] > + > + config_key = 'autocloud.consumer.enabled' > + > + def __init__(self, *args, **kwargs): > + self.supported_archs = [arch for arch, _ in > ComposeJobDetails.ARCH_TYPES] > + > + log.info("Autocloud Consumer is ready for action.") > + super(AutoCloudConsumer, self).__init__(*args, **kwargs) > + > + def consume(self, msg): > + """ This is called when we receive a message matching the topic. """ > + > + log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) > + > + STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) > + VARIANTS_F = ('CloudImages',) > + > + images = [] > + compose_db_update = False > + msg_body = msg['body'] > + status = msg_body['msg']['status'] > + compose_images_json = None > + > + if status in STATUS_F: > + location = msg_body['msg']['location'] > + json_metadata = '{}/metadata/images.json'.format(location) > + resp = requests.get(json_metadata) > + compose_images_json = getattr(resp, 'json', False) > + > + if compose_images_json is not None: > + compose_images_json = compose_images_json() > + compose_images = compose_images_json['payload']['images'] > + compose_details = compose_images_json['payload']['compose'] > + compose_images = dict((variant, compose_images[variant]) > + for variant in VARIANTS_F > + if variant in compose_images) > + compose_id = compose_details['id'] > + rel = fedfind.release.get_release(cid=compose_id) > + release = rel.release > + compose_details.update({'release': release}) > + > + compose_images_variants = [variant for variant in VARIANTS_F > + if variant in compose_images] > + > + for variant in compose_images_variants: > + compose_image = compose_images[variant] > + for arch, payload in compose_image.iteritems(): > + > + if arch not in self.supported_archs: > + continue > + > + for item in payload: > + relative_path = item['path'] > + if not is_valid_image(relative_path): > + continue > + absolute_path = '{}/{}'.format(location, relative_path) > + item.update({ > + 'compose': compose_details, > + 'absolute_path': absolute_path, > + }) > + images.append(item) > + compose_db_update = True > + > + if compose_db_update: > + session = init_model() > + compose_date = > datetime.strptime(compose_details['date'], '%Y%m%d') > + try: > + cd = ComposeDetails( > + date=compose_date, > + compose_id=compose_details['id'], > + respin=compose_details['respin'], > + type=compose_details['type'], > + status=u'q', > + location=location, > + ) > + > + session.add(cd) > + session.commit() > + > + compose_details.update({ > + 'status': 'queued', > + 'compose_job_id': cd.id, > + }) > + publish_to_fedmsg(topic='compose.queued', > + **compose_details) > + except exc.IntegrityError: > + session.rollback() > + cd = session.query(ComposeDetails).filter_by( > + compose_id=compose_details['id']).first() > + log.info('Compose already exists %s: %s' % ( > + compose_details['id'], > + cd.id > + )) > + session.close() > + > + num_images = len(images) > + for pos, image in enumerate(images): > + image.update({'pos': (pos+1, num_images)}) > + > + produce_jobs(images) > + > diff --git a/roles/autocloud/backend/tasks/main.yml > b/roles/autocloud/backend/tasks/main.yml > index 7ff5811..e1024c3 100644 > --- a/roles/autocloud/backend/tasks/main.yml > +++ b/roles/autocloud/backend/tasks/main.yml > @@ -139,3 +139,17 @@ > tags: > - autocloud > - autocloud/backend > + > +# > +# Install hotfix to ignore new architectures > +# See PR - https://github.com/kushaldas/autocloud/pull/62/ > +# > +- name: hotfix - copy over consumer files > + copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }} > + with_items: > + - { src: 'hotfix/autocloud/consumer.py', dest: > '/usr/lib/python2.7/site-packages/autocloud' } > + notify: > + - restart fedmsg-hub > + tags: > + - autocloud > + - hotfix > -- > 2.9.4 > > > From 1403d8118ab9483d0753f1995bec5ce6e78f0df2 Mon Sep 17 00:00:00 2001 > From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> > Date: Mon, 26 Mar 2018 18:53:01 +0530 > Subject: [PATCH 3/3] autocloud, hotfix: Add the patch for the hotfix (F28 > messages) > > Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx> > --- > files/hotfix/autocloud/consumer.py | 12 ++++++++++-- > 1 file changed, 10 insertions(+), 2 deletions(-) > > diff --git a/files/hotfix/autocloud/consumer.py > b/files/hotfix/autocloud/consumer.py > index a7b2aa5..0ff8317 100644 > --- a/files/hotfix/autocloud/consumer.py > +++ b/files/hotfix/autocloud/consumer.py > @@ -48,7 +48,6 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): > log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) > > STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) > - VARIANTS_F = ('CloudImages',) > > images = [] > compose_db_update = False > @@ -56,6 +55,16 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): > status = msg_body['msg']['status'] > compose_images_json = None > > + # Till F27, both cloud-base and atomic images were available > + # under variant CloudImages. With F28 and onward releases, > + # cloud-base image compose moved to cloud variant and atomic images > + # moved under atomic variant. > + prev_rel = ['26', '27'] > + if msg_body['msg']['release_version'] in prev_rel: > + VARIANTS_F = ('CloudImages',) > + else: > + VARIANTS_F = ('AtomicHost', 'Cloud') > + > if status in STATUS_F: > location = msg_body['msg']['location'] > json_metadata = '{}/metadata/images.json'.format(location) > @@ -133,4 +142,3 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): > image.update({'pos': (pos+1, num_images)}) > > produce_jobs(images) > - > -- > 2.9.4 > > > -- > Sayan Chowdhury <https://sayanchowdhury.dgplug.org/> > Senior Software Engineer, Fedora Engineering - Emerging Platform > GPG Fingerprint : 0F16 E841 E517 225C 7D13 AB3C B023 9931 9CD0 5C8B > > > Proud to work at The Open Organization! > _______________________________________________ > infrastructure mailing list -- infrastructure@xxxxxxxxxxxxxxxxxxxxxxx > To unsubscribe send an email to infrastructure-leave@xxxxxxxxxxxxxxxxxxxxxxx -- Stephen J Smoogen. _______________________________________________ infrastructure mailing list -- infrastructure@xxxxxxxxxxxxxxxxxxxxxxx To unsubscribe send an email to infrastructure-leave@xxxxxxxxxxxxxxxxxxxxxxx
