Re: FBR: Fix for processing F28 Cloud & Atomic messages in Autocloud

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



This looks workable +1

On 26 March 2018 at 09:30, Sayan Chowdhury
<sayan.chowdhury2012@xxxxxxxxx> wrote:
> Hi,
>
> This is similar to the fedimg patch sent to the list earlier.
> Autocloud right now is not processing the messages because of the
> change in the F28 messages. This patch would fix the issue.
>
> This also contains a commit to remove a earlier hotfix in autocloud.
> That hotfix was deployed during the last release but I forgot to
> remove the hotfix. Don't know if this is proper but if needed I can
> break this into two FBRs too.
>
> Here is the PR to the corresponding change[0]
>
> Thoughts? +1s?
>
> [0] https://github.com/kushaldas/autocloud/pull/62
>
> From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx>
> Date: Mon, 26 Mar 2018 18:42:33 +0530
> Subject: [PATCH 1/3] autocloud, hotfix: Remove the autocloud hotfix for the
>  PR#56
>
> Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx>
> ---
>  files/hotfix/autocloud/consumer.py     | 135 ---------------------------------
>  roles/autocloud/backend/tasks/main.yml |  14 ----
>  2 files changed, 149 deletions(-)
>  delete mode 100644 files/hotfix/autocloud/consumer.py
>
> diff --git a/files/hotfix/autocloud/consumer.py
> b/files/hotfix/autocloud/consumer.py
> deleted file mode 100644
> index c216553..0000000
> --- a/files/hotfix/autocloud/consumer.py
> +++ /dev/null
> @@ -1,135 +0,0 @@
> -# -*- coding: utf-8 -*-
> -from datetime import datetime
> -
> -import requests
> -import fedmsg.consumers
> -import fedfind.release
> -
> -from sqlalchemy import exc
> -
> -import autocloud
> -
> -from autocloud.models import init_model, ComposeDetails, ComposeJobDetails
> -from autocloud.producer import publish_to_fedmsg
> -from autocloud.utils import is_valid_image, produce_jobs
> -
> -import logging
> -log = logging.getLogger("fedmsg")
> -
> -DEBUG = autocloud.DEBUG
> -
> -
> -class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer):
> -    """
> -    Fedmsg consumer for Autocloud
> -    """
> -
> -    if DEBUG:
> -        topic = [
> -            'org.fedoraproject.dev.__main__.pungi.compose.status.change'
> -        ]
> -
> -    else:
> -        topic = [
> -            'org.fedoraproject.prod.pungi.compose.status.change'
> -        ]
> -
> -    config_key = 'autocloud.consumer.enabled'
> -
> -    def __init__(self, *args, **kwargs):
> -        self.supported_archs = [arch for arch, _ in
> ComposeJobDetails.ARCH_TYPES]
> -
> -        log.info("Autocloud Consumer is ready for action.")
> -        super(AutoCloudConsumer, self).__init__(*args, **kwargs)
> -
> -    def consume(self, msg):
> -        """ This is called when we receive a message matching the topic. """
> -
> -        log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id']))
> -
> -        STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',)
> -        VARIANTS_F = ('CloudImages',)
> -
> -        images = []
> -        compose_db_update = False
> -        msg_body = msg['body']
> -        status = msg_body['msg']['status']
> -        compose_images_json = None
> -
> -        if status in STATUS_F:
> -            location = msg_body['msg']['location']
> -            json_metadata = '{}/metadata/images.json'.format(location)
> -            resp = requests.get(json_metadata)
> -            compose_images_json = getattr(resp, 'json', False)
> -
> -        if compose_images_json is not None:
> -            compose_images_json = compose_images_json()
> -            compose_images = compose_images_json['payload']['images']
> -            compose_details = compose_images_json['payload']['compose']
> -            compose_images = dict((variant, compose_images[variant])
> -                                  for variant in VARIANTS_F
> -                                  if variant in compose_images)
> -            compose_id = compose_details['id']
> -            rel = fedfind.release.get_release(cid=compose_id)
> -            release = rel.release
> -            compose_details.update({'release': release})
> -
> -            compose_images_variants = [variant for variant in VARIANTS_F
> -                                       if variant in compose_images]
> -
> -            for variant in compose_images_variants:
> -                compose_image = compose_images[variant]
> -                for arch, payload in compose_image.iteritems():
> -
> -                    if arch not in self.supported_archs:
> -                        continue
> -
> -                    for item in payload:
> -                        relative_path = item['path']
> -                        if not is_valid_image(relative_path):
> -                            continue
> -                        absolute_path = '{}/{}'.format(location, relative_path)
> -                        item.update({
> -                            'compose': compose_details,
> -                            'absolute_path': absolute_path,
> -                        })
> -                        images.append(item)
> -                        compose_db_update = True
> -
> -            if compose_db_update:
> -                session = init_model()
> -                compose_date =
> datetime.strptime(compose_details['date'], '%Y%m%d')
> -                try:
> -                    cd = ComposeDetails(
> -                        date=compose_date,
> -                        compose_id=compose_details['id'],
> -                        respin=compose_details['respin'],
> -                        type=compose_details['type'],
> -                        status=u'q',
> -                        location=location,
> -                    )
> -
> -                    session.add(cd)
> -                    session.commit()
> -
> -                    compose_details.update({
> -                        'status': 'queued',
> -                        'compose_job_id': cd.id,
> -                    })
> -                    publish_to_fedmsg(topic='compose.queued',
> -                                      **compose_details)
> -                except exc.IntegrityError:
> -                    session.rollback()
> -                    cd = session.query(ComposeDetails).filter_by(
> -                        compose_id=compose_details['id']).first()
> -                    log.info('Compose already exists %s: %s' % (
> -                        compose_details['id'],
> -                        cd.id
> -                    ))
> -                session.close()
> -
> -            num_images = len(images)
> -            for pos, image in enumerate(images):
> -                image.update({'pos': (pos+1, num_images)})
> -
> -            produce_jobs(images)
> diff --git a/roles/autocloud/backend/tasks/main.yml
> b/roles/autocloud/backend/tasks/main.yml
> index bc50df1..7ff5811 100644
> --- a/roles/autocloud/backend/tasks/main.yml
> +++ b/roles/autocloud/backend/tasks/main.yml
> @@ -139,17 +139,3 @@
>    tags:
>    - autocloud
>    - autocloud/backend
> -
> -#
> -# Install hotfix to ignore new architectures
> -# See PR - https://github.com/kushaldas/autocloud/pull/56/
> -#
> -- name: hotfix - copy over consumer files
> -  copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }}
> -  with_items:
> -  - { src: 'hotfix/autocloud/consumer.py', dest:
> '/usr/lib/python2.7/site-packages/autocloud' }
> -  notify:
> -  - restart fedmsg-hub
> -  tags:
> -  - autocloud
> -  - hotfix
> --
> 2.9.4
>
>
> From 7593e3dddaf85011b27c989a3f14c9794f4265bc Mon Sep 17 00:00:00 2001
> From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx>
> Date: Mon, 26 Mar 2018 18:50:42 +0530
> Subject: [PATCH 2/3] autocloud, hotfix: Add the hotfix to process F28 Atomic,
>  Cloud messages
>
> Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx>
> ---
>  files/hotfix/autocloud/consumer.py     | 136 +++++++++++++++++++++++++++++++++
>  roles/autocloud/backend/tasks/main.yml |  14 ++++
>  2 files changed, 150 insertions(+)
>  create mode 100644 files/hotfix/autocloud/consumer.py
>
> diff --git a/files/hotfix/autocloud/consumer.py
> b/files/hotfix/autocloud/consumer.py
> new file mode 100644
> index 0000000..a7b2aa5
> --- /dev/null
> +++ b/files/hotfix/autocloud/consumer.py
> @@ -0,0 +1,136 @@
> +# -*- coding: utf-8 -*-
> +from datetime import datetime
> +
> +import requests
> +import fedmsg.consumers
> +import fedfind.release
> +
> +from sqlalchemy import exc
> +
> +import autocloud
> +
> +from autocloud.models import init_model, ComposeDetails, ComposeJobDetails
> +from autocloud.producer import publish_to_fedmsg
> +from autocloud.utils import is_valid_image, produce_jobs
> +
> +import logging
> +log = logging.getLogger("fedmsg")
> +
> +DEBUG = autocloud.DEBUG
> +
> +
> +class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer):
> +    """
> +    Fedmsg consumer for Autocloud
> +    """
> +
> +    if DEBUG:
> +        topic = [
> +            'org.fedoraproject.dev.__main__.pungi.compose.status.change'
> +        ]
> +
> +    else:
> +        topic = [
> +            'org.fedoraproject.prod.pungi.compose.status.change'
> +        ]
> +
> +    config_key = 'autocloud.consumer.enabled'
> +
> +    def __init__(self, *args, **kwargs):
> +        self.supported_archs = [arch for arch, _ in
> ComposeJobDetails.ARCH_TYPES]
> +
> +        log.info("Autocloud Consumer is ready for action.")
> +        super(AutoCloudConsumer, self).__init__(*args, **kwargs)
> +
> +    def consume(self, msg):
> +        """ This is called when we receive a message matching the topic. """
> +
> +        log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id']))
> +
> +        STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',)
> +        VARIANTS_F = ('CloudImages',)
> +
> +        images = []
> +        compose_db_update = False
> +        msg_body = msg['body']
> +        status = msg_body['msg']['status']
> +        compose_images_json = None
> +
> +        if status in STATUS_F:
> +            location = msg_body['msg']['location']
> +            json_metadata = '{}/metadata/images.json'.format(location)
> +            resp = requests.get(json_metadata)
> +            compose_images_json = getattr(resp, 'json', False)
> +
> +        if compose_images_json is not None:
> +            compose_images_json = compose_images_json()
> +            compose_images = compose_images_json['payload']['images']
> +            compose_details = compose_images_json['payload']['compose']
> +            compose_images = dict((variant, compose_images[variant])
> +                                  for variant in VARIANTS_F
> +                                  if variant in compose_images)
> +            compose_id = compose_details['id']
> +            rel = fedfind.release.get_release(cid=compose_id)
> +            release = rel.release
> +            compose_details.update({'release': release})
> +
> +            compose_images_variants = [variant for variant in VARIANTS_F
> +                                       if variant in compose_images]
> +
> +            for variant in compose_images_variants:
> +                compose_image = compose_images[variant]
> +                for arch, payload in compose_image.iteritems():
> +
> +                    if arch not in self.supported_archs:
> +                        continue
> +
> +                    for item in payload:
> +                        relative_path = item['path']
> +                        if not is_valid_image(relative_path):
> +                            continue
> +                        absolute_path = '{}/{}'.format(location, relative_path)
> +                        item.update({
> +                            'compose': compose_details,
> +                            'absolute_path': absolute_path,
> +                        })
> +                        images.append(item)
> +                        compose_db_update = True
> +
> +            if compose_db_update:
> +                session = init_model()
> +                compose_date =
> datetime.strptime(compose_details['date'], '%Y%m%d')
> +                try:
> +                    cd = ComposeDetails(
> +                        date=compose_date,
> +                        compose_id=compose_details['id'],
> +                        respin=compose_details['respin'],
> +                        type=compose_details['type'],
> +                        status=u'q',
> +                        location=location,
> +                    )
> +
> +                    session.add(cd)
> +                    session.commit()
> +
> +                    compose_details.update({
> +                        'status': 'queued',
> +                        'compose_job_id': cd.id,
> +                    })
> +                    publish_to_fedmsg(topic='compose.queued',
> +                                      **compose_details)
> +                except exc.IntegrityError:
> +                    session.rollback()
> +                    cd = session.query(ComposeDetails).filter_by(
> +                        compose_id=compose_details['id']).first()
> +                    log.info('Compose already exists %s: %s' % (
> +                        compose_details['id'],
> +                        cd.id
> +                    ))
> +                session.close()
> +
> +            num_images = len(images)
> +            for pos, image in enumerate(images):
> +                image.update({'pos': (pos+1, num_images)})
> +
> +            produce_jobs(images)
> +
> diff --git a/roles/autocloud/backend/tasks/main.yml
> b/roles/autocloud/backend/tasks/main.yml
> index 7ff5811..e1024c3 100644
> --- a/roles/autocloud/backend/tasks/main.yml
> +++ b/roles/autocloud/backend/tasks/main.yml
> @@ -139,3 +139,17 @@
>    tags:
>    - autocloud
>    - autocloud/backend
> +
> +#
> +# Install hotfix to ignore new architectures
> +# See PR - https://github.com/kushaldas/autocloud/pull/62/
> +#
> +- name: hotfix - copy over consumer files
> +  copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }}
> +  with_items:
> +  - { src: 'hotfix/autocloud/consumer.py', dest:
> '/usr/lib/python2.7/site-packages/autocloud' }
> +  notify:
> +  - restart fedmsg-hub
> +  tags:
> +  - autocloud
> +  - hotfix
> --
> 2.9.4
>
>
> From 1403d8118ab9483d0753f1995bec5ce6e78f0df2 Mon Sep 17 00:00:00 2001
> From: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx>
> Date: Mon, 26 Mar 2018 18:53:01 +0530
> Subject: [PATCH 3/3] autocloud, hotfix: Add the patch for the hotfix (F28
>  messages)
>
> Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@xxxxxxxxx>
> ---
>  files/hotfix/autocloud/consumer.py | 12 ++++++++++--
>  1 file changed, 10 insertions(+), 2 deletions(-)
>
> diff --git a/files/hotfix/autocloud/consumer.py
> b/files/hotfix/autocloud/consumer.py
> index a7b2aa5..0ff8317 100644
> --- a/files/hotfix/autocloud/consumer.py
> +++ b/files/hotfix/autocloud/consumer.py
> @@ -48,7 +48,6 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer):
>          log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id']))
>
>          STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',)
> -        VARIANTS_F = ('CloudImages',)
>
>          images = []
>          compose_db_update = False
> @@ -56,6 +55,16 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer):
>          status = msg_body['msg']['status']
>          compose_images_json = None
>
> +        # Till F27, both cloud-base and atomic images were available
> +        # under variant CloudImages. With F28 and onward releases,
> +        # cloud-base image compose moved to cloud variant and atomic images
> +        # moved under atomic variant.
> +        prev_rel = ['26', '27']
> +        if msg_body['msg']['release_version'] in prev_rel:
> +            VARIANTS_F = ('CloudImages',)
> +        else:
> +            VARIANTS_F = ('AtomicHost', 'Cloud')
> +
>          if status in STATUS_F:
>              location = msg_body['msg']['location']
>              json_metadata = '{}/metadata/images.json'.format(location)
> @@ -133,4 +142,3 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer):
>                  image.update({'pos': (pos+1, num_images)})
>
>              produce_jobs(images)
> -
> --
> 2.9.4
>
>
> --
> Sayan Chowdhury <https://sayanchowdhury.dgplug.org/>
> Senior Software Engineer, Fedora Engineering - Emerging Platform
> GPG Fingerprint : 0F16 E841 E517 225C 7D13  AB3C B023 9931 9CD0 5C8B
>
>
> Proud to work at The Open Organization!
> _______________________________________________
> infrastructure mailing list -- infrastructure@xxxxxxxxxxxxxxxxxxxxxxx
> To unsubscribe send an email to infrastructure-leave@xxxxxxxxxxxxxxxxxxxxxxx



-- 
Stephen J Smoogen.
_______________________________________________
infrastructure mailing list -- infrastructure@xxxxxxxxxxxxxxxxxxxxxxx
To unsubscribe send an email to infrastructure-leave@xxxxxxxxxxxxxxxxxxxxxxx




[Index of Archives]     [Fedora Development]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Yosemite News]     [KDE Users]

  Powered by Linux