Hi,
There have been 2 comments about my patches (exception handling and
inline dicts) and I've addressed both of them. I'll attach the latest
set of patches again to avoid confusion and would be happy if someone
with git access could merge them.
They are also available in the fedora-messaging branch of my ansible
checkout on batcave if that makes it easier to merge them.
Thanks a lot
Karsten
Am 09.11.19 um 01:38 schrieb karsten@xxxxxxxxxxxxxxxxx:
Hello,
There is work in progress to migrate our applications from the current
message bus 'fedmsg' to the AMPG based 'fedora-messaging'.
Attached are a couple of patches that prepare our ansible scripts for
this.
Please review those patches and comment, thanks !
Karsten
>From 74d53a7039ce07c122b105d32d0e7b41df71b004 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Wed, 13 Nov 2019 11:23:44 +0100
Subject: [PATCH 15/15] don't use inline dict for message body
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
callback_plugins/fedora_messaging_callback.py | 24 ++++++++++---------
.../fedora_messaging_callback2.py | 24 ++++++++++---------
2 files changed, 26 insertions(+), 22 deletions(-)
diff --git a/callback_plugins/fedora_messaging_callback.py b/callback_plugins/fedora_messaging_callback.py
index 372080212..620fe456e 100644
--- a/callback_plugins/fedora_messaging_callback.py
+++ b/callback_plugins/fedora_messaging_callback.py
@@ -64,14 +64,14 @@ class CallbackModule(CallbackBase):
try:
msg = Message(
topic="ansible.playbook.start",
- body=dict(
- playbook=path,
- userid=getlogin(),
- extra_vars=play.playbook.extra_vars,
- inventory=play.playbook.inventory.host_list,
- playbook_checksum=play.playbook.check,
- check=play.playbook.check,
- ),
+ body={
+ 'playbook': path,
+ 'userid': getlogin(),
+ 'extra_vars': play.playbook.extra_vars,
+ 'inventory': play.playbook.inventory.host_list,
+ 'playbook_checksum': play.playbook.check,
+ 'check': play.playbook.check
+ }
)
publish(msg)
except PublishReturned as e:
@@ -90,9 +90,11 @@ class CallbackModule(CallbackBase):
try:
msg = Message(
topic="ansible.playbook.complete",
- body=dict(
- playbook=self.playbook_path, userid=getlogin(), results=results
- ),
+ body={
+ 'playbook': self.playbook_path,
+ 'userid': getlogin(),
+ 'results': results
+ )
)
publish(msg)
except PublishReturned as e:
diff --git a/callback_plugins/fedora_messaging_callback2.py b/callback_plugins/fedora_messaging_callback2.py
index 0b08d6b36..dc4176376 100644
--- a/callback_plugins/fedora_messaging_callback2.py
+++ b/callback_plugins/fedora_messaging_callback2.py
@@ -82,14 +82,14 @@ class CallbackModule(CallbackBase):
try:
msg = Message(
topic="ansible.playbook.start",
- body=dict(
- playbook=path,
- userid=getlogin(),
- extra_vars=play._variable_manager.extra_vars,
- inventory=play._variable_manager._inventory._sources,
- playbook_checksum=secure_hash(path),
- check=self.play_context.check_mode,
- ),
+ body={
+ 'playbook': path,
+ 'userid': getlogin(),
+ 'extra_vars': play._variable_manager.extra_vars,
+ 'inventory': play._variable_manager._inventory._sources,
+ 'playbook_checksum': secure_hash(path),
+ 'check': self.play_context.check_mode
+ }
)
publish(msg)
except PublishReturned as e:
@@ -108,9 +108,11 @@ class CallbackModule(CallbackBase):
try:
msg = Message(
topic="ansible.playbook.complete",
- body=dict(
- playbook=self.playbook_path, userid=getlogin(), results=results
- ),
+ body={
+ 'playbook': self.playbook_path,
+ 'userid': getlogin(),
+ 'results': results
+ }
)
publish(msg)
except PublishReturned as e:
--
2.21.0
>From eb7a799f0b98f2752762facaffb2909eefa9d41c Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Tue, 12 Nov 2019 14:47:22 +0100
Subject: [PATCH 14/15] add exception handling for fedora-messaging callbacks
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
callback_plugins/fedora_messaging_callback.py | 50 ++++++++++++-------
.../fedora_messaging_callback2.py | 50 ++++++++++++-------
2 files changed, 66 insertions(+), 34 deletions(-)
diff --git a/callback_plugins/fedora_messaging_callback.py b/callback_plugins/fedora_messaging_callback.py
index 31e918dde..372080212 100644
--- a/callback_plugins/fedora_messaging_callback.py
+++ b/callback_plugins/fedora_messaging_callback.py
@@ -19,6 +19,7 @@
import os
import pwd
+import logging
from fedora_messaging.api import Message, publish
from fedora_messaging.exceptions import PublishReturned, ConnectionException
@@ -29,6 +30,7 @@ except ImportError:
# Ansible v1 compat
CallbackBase = object
+LOGGER = logging.getLogger(__name__)
def getlogin():
try:
@@ -59,18 +61,25 @@ class CallbackModule(CallbackBase):
return
if not self.playbook_path:
- msg = Message(
- topic="ansible.playbook.start",
- body=dict(
- playbook=path,
- userid=getlogin(),
- extra_vars=play.playbook.extra_vars,
- inventory=play.playbook.inventory.host_list,
- playbook_checksum=play.playbook.check,
- check=play.playbook.check,
- ),
- )
- publish(msg)
+ try:
+ msg = Message(
+ topic="ansible.playbook.start",
+ body=dict(
+ playbook=path,
+ userid=getlogin(),
+ extra_vars=play.playbook.extra_vars,
+ inventory=play.playbook.inventory.host_list,
+ playbook_checksum=play.playbook.check,
+ check=play.playbook.check,
+ ),
+ )
+ publish(msg)
+ except PublishReturned as e:
+ LOGGER.warning(
+ "Fedora Messaging broker rejected message %s: %s", msg.id, e
+ )
+ except ConnectionException as e:
+ LOGGER.warning("Error sending message %s: %s", msg.id, e)
self.playbook_path = path
def playbook_on_stats(self, stats):
@@ -78,8 +87,15 @@ class CallbackModule(CallbackBase):
return
results = dict([(h, stats.summarize(h)) for h in stats.processed])
- msg = Message(
- topic="ansible.playbook.complete",
- body=dict(playbook=self.playbook_path, userid=getlogin(), results=results),
- )
- publish(msg)
+ try:
+ msg = Message(
+ topic="ansible.playbook.complete",
+ body=dict(
+ playbook=self.playbook_path, userid=getlogin(), results=results
+ ),
+ )
+ publish(msg)
+ except PublishReturned as e:
+ LOGGER.warning("Fedora Messaging broker rejected message %s: %s", msg.id, e)
+ except ConnectionException as e:
+ LOGGER.warning("Error sending message %s: %s", msg.id, e)
diff --git a/callback_plugins/fedora_messaging_callback2.py b/callback_plugins/fedora_messaging_callback2.py
index 3d1e357ac..0b08d6b36 100644
--- a/callback_plugins/fedora_messaging_callback2.py
+++ b/callback_plugins/fedora_messaging_callback2.py
@@ -19,6 +19,7 @@
import os
import pwd
+import logging
from fedora_messaging.api import Message, publish
from fedora_messaging.exceptions import PublishReturned, ConnectionException
@@ -34,6 +35,7 @@ try:
except ImportError:
from ansible.utils import md5 as secure_hash
+LOGGER = logging.getLogger(__name__)
def getlogin():
try:
@@ -77,18 +79,25 @@ class CallbackModule(CallbackBase):
return
if not self.playbook_path:
- msg = Message(
- topic="ansible.playbook.start",
- body=dict(
- playbook=path,
- userid=getlogin(),
- extra_vars=play._variable_manager.extra_vars,
- inventory=play._variable_manager._inventory._sources,
- playbook_checksum=secure_hash(path),
- check=self.play_context.check_mode,
- ),
- )
- publish(msg)
+ try:
+ msg = Message(
+ topic="ansible.playbook.start",
+ body=dict(
+ playbook=path,
+ userid=getlogin(),
+ extra_vars=play._variable_manager.extra_vars,
+ inventory=play._variable_manager._inventory._sources,
+ playbook_checksum=secure_hash(path),
+ check=self.play_context.check_mode,
+ ),
+ )
+ publish(msg)
+ except PublishReturned as e:
+ LOGGER.warning(
+ "Fedora Messaging broker rejected message %s: %s", msg.id, e
+ )
+ except ConnectionException as e:
+ LOGGER.warning("Error sending message %s: %s", msg.id, e)
self.playbook_path = path
def v2_playbook_on_stats(self, stats):
@@ -96,8 +105,15 @@ class CallbackModule(CallbackBase):
return
results = dict([(h, stats.summarize(h)) for h in stats.processed])
- msg = Message(
- topic="ansible.playbook.complete",
- body=dict(playbook=self.playbook_path, userid=getlogin(), results=results),
- )
- publish(msg)
+ try:
+ msg = Message(
+ topic="ansible.playbook.complete",
+ body=dict(
+ playbook=self.playbook_path, userid=getlogin(), results=results
+ ),
+ )
+ publish(msg)
+ except PublishReturned as e:
+ LOGGER.warning("Fedora Messaging broker rejected message %s: %s", msg.id, e)
+ except ConnectionException as e:
+ LOGGER.warning("Error sending message %s: %s", msg.id, e)
--
2.21.0
>From 1e9acd820dfb57ca5931046209f1da8f0bc649e8 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 22:34:47 +0100
Subject: [PATCH 13/15] add callbacks for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
callback_plugins/fedora_messaging_callback.py | 85 +++++++++++++++
.../fedora_messaging_callback2.py | 103 ++++++++++++++++++
2 files changed, 188 insertions(+)
create mode 100644 callback_plugins/fedora_messaging_callback.py
create mode 100644 callback_plugins/fedora_messaging_callback2.py
diff --git a/callback_plugins/fedora_messaging_callback.py b/callback_plugins/fedora_messaging_callback.py
new file mode 100644
index 000000000..31e918dde
--- /dev/null
+++ b/callback_plugins/fedora_messaging_callback.py
@@ -0,0 +1,85 @@
+# (C) 2012, Michael DeHaan, <michael.dehaan@xxxxxxxxx>
+# based on the log_plays example
+# skvidal@xxxxxxxxxxxxxxxxx
+# rbean@xxxxxxxxxx
+# karsten@xxxxxxxxxx changes for fedora-messaging
+
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import pwd
+
+from fedora_messaging.api import Message, publish
+from fedora_messaging.exceptions import PublishReturned, ConnectionException
+
+try:
+ from ansible.plugins.callback import CallbackBase
+except ImportError:
+ # Ansible v1 compat
+ CallbackBase = object
+
+
+def getlogin():
+ try:
+ user = os.getlogin()
+ except OSError as e:
+ user = pwd.getpwuid(os.geteuid())[0]
+ return user
+
+
+class CallbackModule(CallbackBase):
+ """ Publish playbook starts and stops to fedora-messaging. """
+
+ playbook_path = None
+
+ def __init__(self):
+ pass
+
+ def playbook_on_play_start(self, pattern):
+ # This gets called once for each play.. but we just issue a message once
+ # for the first one. One per "playbook"
+ play = getattr(self, "play", None)
+ if play:
+ # figure out where the playbook FILE is
+ path = os.path.abspath(play.playbook.filename)
+
+ # Bail out early without publishing if we're in --check mode
+ if play.playbook.check:
+ return
+
+ if not self.playbook_path:
+ msg = Message(
+ topic="ansible.playbook.start",
+ body=dict(
+ playbook=path,
+ userid=getlogin(),
+ extra_vars=play.playbook.extra_vars,
+ inventory=play.playbook.inventory.host_list,
+ playbook_checksum=play.playbook.check,
+ check=play.playbook.check,
+ ),
+ )
+ publish(msg)
+ self.playbook_path = path
+
+ def playbook_on_stats(self, stats):
+ if not self.playbook_path:
+ return
+
+ results = dict([(h, stats.summarize(h)) for h in stats.processed])
+ msg = Message(
+ topic="ansible.playbook.complete",
+ body=dict(playbook=self.playbook_path, userid=getlogin(), results=results),
+ )
+ publish(msg)
diff --git a/callback_plugins/fedora_messaging_callback2.py b/callback_plugins/fedora_messaging_callback2.py
new file mode 100644
index 000000000..3d1e357ac
--- /dev/null
+++ b/callback_plugins/fedora_messaging_callback2.py
@@ -0,0 +1,103 @@
+# (C) 2012, Michael DeHaan, <michael.dehaan@xxxxxxxxx>
+# based on the log_plays example
+# skvidal@xxxxxxxxxxxxxxxxx
+# rbean@xxxxxxxxxx
+# karsten@xxxxxxxxxx changes for fedora-messaging
+
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import pwd
+
+from fedora_messaging.api import Message, publish
+from fedora_messaging.exceptions import PublishReturned, ConnectionException
+
+try:
+ from ansible.plugins.callback import CallbackBase
+except ImportError:
+ # Ansible v1 compat
+ CallbackBase = object
+
+try:
+ from ansible.utils.hashing import secure_hash
+except ImportError:
+ from ansible.utils import md5 as secure_hash
+
+
+def getlogin():
+ try:
+ user = os.getlogin()
+ except OSError as e:
+ user = pwd.getpwuid(os.geteuid())[0]
+ return user
+
+
+class CallbackModule(CallbackBase):
+ """ Publish playbook starts and stops to fedora_messaging. """
+
+ CALLBACK_NAME = "fedora_messaging_callback2"
+ CALLBACK_TYPE = "notification"
+ CALLBACK_VERSION = 2.0
+ CALLBACK_NEEDS_WHITELIST = True
+
+ playbook_path = None
+
+ def __init__(self):
+ self.play = None
+ self.playbook = None
+
+ super(CallbackModule, self).__init__()
+
+ def set_play_context(self, play_context):
+ self.play_context = play_context
+
+ def v2_playbook_on_start(self, playbook):
+ self.playbook = playbook
+
+ def v2_playbook_on_play_start(self, play):
+ # This gets called once for each play.. but we just issue a message once
+ # for the first one. One per "playbook"
+ if self.playbook:
+ # figure out where the playbook FILE is
+ path = os.path.abspath(self.playbook._file_name)
+
+ # Bail out early without publishing if we're in --check mode
+ if self.play_context.check_mode:
+ return
+
+ if not self.playbook_path:
+ msg = Message(
+ topic="ansible.playbook.start",
+ body=dict(
+ playbook=path,
+ userid=getlogin(),
+ extra_vars=play._variable_manager.extra_vars,
+ inventory=play._variable_manager._inventory._sources,
+ playbook_checksum=secure_hash(path),
+ check=self.play_context.check_mode,
+ ),
+ )
+ publish(msg)
+ self.playbook_path = path
+
+ def v2_playbook_on_stats(self, stats):
+ if not self.playbook_path:
+ return
+
+ results = dict([(h, stats.summarize(h)) for h in stats.processed])
+ msg = Message(
+ topic="ansible.playbook.complete",
+ body=dict(playbook=self.playbook_path, userid=getlogin(), results=results),
+ )
+ publish(msg)
--
2.21.0
>From 0a27afb0424efed2757dff73379425faf3855211 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 22:11:30 +0100
Subject: [PATCH 12/15] prepare notifs-backend, notifs-web for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/notifs-backend.yml | 6 +++++-
playbooks/groups/notifs-web.yml | 6 +++++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/playbooks/groups/notifs-backend.yml b/playbooks/groups/notifs-backend.yml
index 93df0c2f8..b33c28744 100644
--- a/playbooks/groups/notifs-backend.yml
+++ b/playbooks/groups/notifs-backend.yml
@@ -25,7 +25,11 @@
- fas_client
- nagios_client
- collectd/base
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "notifs-backend{{ env_suffix }}"}
- sudo
# The proxies don't actually need to talk to these hosts so we won't bother
# putting them on the vpn.
diff --git a/playbooks/groups/notifs-web.yml b/playbooks/groups/notifs-web.yml
index ec0e963a3..2a6ca8668 100644
--- a/playbooks/groups/notifs-web.yml
+++ b/playbooks/groups/notifs-web.yml
@@ -23,7 +23,11 @@
- fas_client
- collectd/base
- mod_wsgi
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "notifs-web{{ env_suffix }}"}
- notifs/frontend
- sudo
- { role: openvpn/client,
--
2.21.0
>From 14cc7db58ffa07439dfb6dba3b61fc43d735a2af Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 22:08:46 +0100
Subject: [PATCH 11/15] prepare zanata for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/zanata2fedmsg.yml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/zanata2fedmsg.yml b/playbooks/groups/zanata2fedmsg.yml
index 0694295ef..5885b0e2a 100644
--- a/playbooks/groups/zanata2fedmsg.yml
+++ b/playbooks/groups/zanata2fedmsg.yml
@@ -53,4 +53,8 @@
roles:
- zanata2fedmsg
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "zanata{{ env_suffix }}"}
--
2.21.0
>From b91ab939f0c303491b47b2b037d2c929cb8c9559 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 22:05:26 +0100
Subject: [PATCH 10/15] prepare happiness* for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
.../hosts/happinesspackets-stg.fedorainfracloud.org.yml | 6 +++++-
playbooks/hosts/happinesspackets.fedorainfracloud.org.yml | 6 +++++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml b/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml
index f146c40f1..20e7c2043 100644
--- a/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml
+++ b/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml
@@ -34,7 +34,11 @@
roles:
- basessh
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "happipstgfedorainfracloud{{ env_suffix }}"}
- { role: letsencrypt, site_name: 'happinesspackets-stg.fedorainfracloud.org' }
handlers:
diff --git a/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml b/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml
index 2cd1acd56..a57e047e3 100644
--- a/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml
+++ b/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml
@@ -34,7 +34,11 @@
roles:
- basessh
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "happipfedorainfracloud{{ env_suffix }}"}
- { role: letsencrypt, site_name: 'happinesspackets.fedorainfracloud.org' }
handlers:
--
2.21.0
>From d54ca4d8e0d65bd5e01325ea184c64f888d5d771 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 22:01:38 +0100
Subject: [PATCH 09/15] prepare value for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/value.yml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/value.yml b/playbooks/groups/value.yml
index 8e2dd01df..cef0765c6 100644
--- a/playbooks/groups/value.yml
+++ b/playbooks/groups/value.yml
@@ -18,7 +18,11 @@
- fas_client
- collectd/base
- apache
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "value{{ env_suffix }}"}
- fedmsg/irc
- supybot
- sudo
--
2.21.0
>From 7b5aac944045a9a5d3d701a4d35678402c99a5ec Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 21:59:37 +0100
Subject: [PATCH 08/15] prepare mirrormanager for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/mirrormanager.yml | 6 +++-
roles/mirrormanager/backend/tasks/main.yml | 35 ++++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/mirrormanager.yml b/playbooks/groups/mirrormanager.yml
index c5a7722f4..6e73a6216 100644
--- a/playbooks/groups/mirrormanager.yml
+++ b/playbooks/groups/mirrormanager.yml
@@ -100,7 +100,11 @@
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
roles:
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "mirrormanager{{ env_suffix }}"}
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
diff --git a/roles/mirrormanager/backend/tasks/main.yml b/roles/mirrormanager/backend/tasks/main.yml
index 3793f6ffe..20fff62ab 100644
--- a/roles/mirrormanager/backend/tasks/main.yml
+++ b/roles/mirrormanager/backend/tasks/main.yml
@@ -9,6 +9,7 @@
- bzip2
- python-psycopg2
- fedmsg
+ - fedora-messaging
- jq
- geolite2-city
- geolite2-country
@@ -86,6 +87,40 @@
- config
when: env != 'staging'
+- name: Create /etc/pki/fedora-messaging
+ file:
+ dest: /etc/pki/fedora-messaging
+ mode: 0775
+ owner: root
+ group: root
+ state: directory
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+# FIXME: do we need to create a mirrormanager cert ?
+- name: Deploy the Fedora mirrormanager fedora-messaging cert
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mirrormanager{{env_suffix}}.crt"
+ dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-cert.pem
+ mode: 0644
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+- name: Deploy the Fedora infra fedora-messaging key
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mirrormanager{{env_suffix}}.key"
+ dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-key.pem
+ mode: 0640
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
+
# To decrease the crawl duration on the mirrors we have been
# recommending to lower the default value of vfs_cache_pressure
# from 100 to 10. This causes the kernel to prefer to keep dentries
--
2.21.0
>From 8690713bd943cb9b50c0f92e524eafadcb3e2a07 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 21:56:28 +0100
Subject: [PATCH 07/15] prepare pdc for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/pdc.yml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/pdc.yml b/playbooks/groups/pdc.yml
index b581507a6..e7f7fea98 100644
--- a/playbooks/groups/pdc.yml
+++ b/playbooks/groups/pdc.yml
@@ -44,7 +44,11 @@
- role: openvpn/client
when: env != "staging"
- mod_wsgi
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "pdc{{ env_suffix }}"}
- pdc/frontend
- name: stuff just for the backend nodes
--
2.21.0
>From d3463a950f1b5e2a908a95a738a82daee77097ee Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 15:32:05 +0100
Subject: [PATCH 06/15] prepare mailman for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/mailman.yml | 6 +++++-
roles/mailman/tasks/main.yml | 35 +++++++++++++++++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/mailman.yml b/playbooks/groups/mailman.yml
index 7bfce6a85..df3e83459 100644
--- a/playbooks/groups/mailman.yml
+++ b/playbooks/groups/mailman.yml
@@ -98,7 +98,11 @@
mailman_hyperkitty_admin_db_pass: "{{ mailman_hk_admin_db_pass }}"
mailman_hyperkitty_db_pass: "{{ mailman_hk_db_pass }}"
mailman_hyperkitty_cookie_key: "{{ mailman_hk_cookie_key }}"
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "mailman{{ env_suffix }}"}
tasks:
- name: install more needed packages
diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml
index 81fd7106a..2109fea97 100644
--- a/roles/mailman/tasks/main.yml
+++ b/roles/mailman/tasks/main.yml
@@ -115,6 +115,7 @@
- python34-PyYAML
# mailman soft dep to convert html to plaintext
- lynx
+ - fedora-messaging
tags:
- packages
- mailman
@@ -554,3 +555,37 @@
- webui-warm-up-cache
tags: mailman
when: inventory_hostname.startswith('mailman01.phx2') or inventory_hostname.startswith('lists-dev')
+
+- name: Create /etc/pki/fedora-messaging
+ file:
+ dest: /etc/pki/fedora-messaging
+ mode: 0775
+ owner: root
+ group: root
+ state: directory
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+# FIXME: Need to create a mailman cert
+- name: Deploy the Fedora mailman fedora-messaging cert
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mailman{{env_suffix}}.crt"
+ dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-cert.pem
+ mode: 0644
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+- name: Deploy the Fedora infra fedora-messaging key
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mailman{{env_suffix}}.key"
+ dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-key.pem
+ mode: 0640
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
--
2.21.0
>From 9b59150553fbd808a59312a7184d4c3741e3ff2c Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Fri, 8 Nov 2019 15:11:15 +0100
Subject: [PATCH 05/15] prepare datagrepper for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/datagrepper.yml | 5 ++++-
roles/datagrepper/tasks/main.yml | 33 ++++++++++++++++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/datagrepper.yml b/playbooks/groups/datagrepper.yml
index 3d87af188..f2e2e31e1 100644
--- a/playbooks/groups/datagrepper.yml
+++ b/playbooks/groups/datagrepper.yml
@@ -19,7 +19,10 @@
- hosts
- fas_client
- collectd/base
- - fedmsg/base
+ - { role: fedmsg/base,
+ when: deployment_type == "prod" }
+ - { role: rabbit/user,
+ username: "datagrepper{{ env_suffix }}"}
- rsyncd
- sudo
- { role: openvpn/client,
diff --git a/roles/datagrepper/tasks/main.yml b/roles/datagrepper/tasks/main.yml
index ccf35a3d0..e13497076 100644
--- a/roles/datagrepper/tasks/main.yml
+++ b/roles/datagrepper/tasks/main.yml
@@ -3,6 +3,7 @@
with_items:
- datagrepper
- python-psycopg2
+ - fedora-messaging
tags:
- packages
- datagrepper
@@ -71,3 +72,35 @@
# selinux policy has been intentionally omitted since that is obtained from fedmsg/base
+- name: Create /etc/pki/fedora-messaging
+ file:
+ dest: /etc/pki/fedora-messaging
+ mode: 0775
+ owner: root
+ group: root
+ state: directory
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+- name: Deploy the Fedora datagrepper fedora-messaging cert
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/datagrepper{{env_suffix}}.crt"
+ dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-cert.pem
+ mode: 0644
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+- name: Deploy the Fedora datagrepper fedora-messaging key
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/datagrepper{{env_suffix}}.key"
+ dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-key.pem
+ mode: 0640
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
--
2.21.0
>From a21e6995e47b13123f2aebf2e008474a9571a879 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Thu, 7 Nov 2019 14:44:19 +0100
Subject: [PATCH 04/15] prepare github2fedmsg for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/github2fedmsg.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/github2fedmsg.yml b/playbooks/groups/github2fedmsg.yml
index 4c208c824..129a58bf5 100644
--- a/playbooks/groups/github2fedmsg.yml
+++ b/playbooks/groups/github2fedmsg.yml
@@ -53,4 +53,5 @@
roles:
- github2fedmsg
- - fedmsg/base
+ - { role: fedmsg/base, when: deployment_type == "prod" }
+ - { role: rabbit/user, when: deployment_type == "stg" }
--
2.21.0
>From bad0902cbe31ce3e5335f20e40b33ec92dc4178c Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Thu, 7 Nov 2019 14:09:17 +0100
Subject: [PATCH 03/15] prepare noc for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/noc.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/noc.yml b/playbooks/groups/noc.yml
index f016091d4..db793bdb7 100644
--- a/playbooks/groups/noc.yml
+++ b/playbooks/groups/noc.yml
@@ -64,7 +64,8 @@
- { role: dhcp_server, when: datacenter == 'phx2' }
- { role: tftp_server, when: datacenter == 'phx2' }
- nagios_server
- - fedmsg/base
+ - { role: fedmsg/base, when: deployment_type == "prod" }
+ - { role: rabbit/user, when: deployment_type == "stg" }
tasks:
- name: install some packages which arent in playbooks
--
2.21.0
>From 1e4176484db23ef434e3567e56f4aee3e8f2765a Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Thu, 7 Nov 2019 14:04:59 +0100
Subject: [PATCH 02/15] prepare sundries for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/sundries.yml | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/playbooks/groups/sundries.yml b/playbooks/groups/sundries.yml
index 561dcaae6..88fbb781c 100644
--- a/playbooks/groups/sundries.yml
+++ b/playbooks/groups/sundries.yml
@@ -51,6 +51,15 @@
when: master_sundries_node|bool
- role: developer/build
when: master_sundries_node|bool
+ - { role: fedmsg/base,
+ when:
+ - master_sundries_node|bool
+ - deployment_type == "prod" }
+ - { role: rabbit/user,
+ username: "sundries{{ env_suffix }}",
+ when:
+ - master_sundries_node|bool
+ - deployment_type == "stg" }
- role: fedmsg/base
when: master_sundries_node|bool
- role: nfs/client
--
2.21.0
>From 2b3815d55dac9f2d2dc77435dc9120f1873c5e13 Mon Sep 17 00:00:00 2001
From: Karsten Hopp <karsten@xxxxxxxxxx>
Date: Thu, 7 Nov 2019 13:52:31 +0100
Subject: [PATCH 01/15] prepare wiki for fedora-messaging
Signed-off-by: Karsten Hopp <karsten@xxxxxxxxxx>
---
playbooks/groups/wiki.yml | 8 +++-
roles/mediawiki/tasks/main.yml | 41 +++++++++++++++++++
.../templates/LocalSettings.php.fp.j2 | 1 +
3 files changed, 49 insertions(+), 1 deletion(-)
diff --git a/playbooks/groups/wiki.yml b/playbooks/groups/wiki.yml
index b3f4b7ece..106e583dc 100644
--- a/playbooks/groups/wiki.yml
+++ b/playbooks/groups/wiki.yml
@@ -26,7 +26,13 @@
- fas_client
- collectd/base
- apache
- - fedmsg/base
+ - { fedmsg/base,
+ when: deployment_type == "prod" }
+ # Set up for fedora-messaging
+ - { role: rabbit/user,
+ username: "wiki{{ env_suffix }}"}
+ - role: rabbit/queue
+ username: "wiki{{ env_suffix }}"
- { role: nfs/client, when: env == "staging", mnt_dir: '/mnt/web/attachments', nfs_src_dir: 'fedora_app_staging/app/attachments' }
- { role: nfs/client, when: env != "staging", mnt_dir: '/mnt/web/attachments', nfs_src_dir: 'fedora_app/app/attachments' }
- mediawiki
diff --git a/roles/mediawiki/tasks/main.yml b/roles/mediawiki/tasks/main.yml
index 958782dd8..843ae4358 100644
--- a/roles/mediawiki/tasks/main.yml
+++ b/roles/mediawiki/tasks/main.yml
@@ -72,6 +72,47 @@
- config
- mediawiki
+#- name: adding fedora-messaging emit
+# copy: src=fedora-message-emit.php dest=/usr/share/{{ wikiver }}/extensions/fedora-messaging-emit.php owner=root group=root mode=775
+# tags:
+# - config
+# - mediawiki
+
+- name: Create /etc/pki/fedora-messaging
+ file:
+ dest: /etc/pki/fedora-messaging
+ mode: 0775
+ owner: root
+ group: root
+ state: directory
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+# FIXME: We currently don't seem to have a wiki cert, need to create one
+- name: Deploy the Fedora wiki fedora-messaging cert
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mediawiki{{env_suffix}}.crt"
+ dest: /etc/pki/fedora-messaging/mediawiki{{env_suffix}}-cert.pem
+ mode: 0644
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
+
+# FIXME: We currently don't seem to have a wiki key, need to create one
+- name: Deploy the Fedora wiki fedora-messaging key
+ copy:
+ src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mediawiki{{env_suffix}}.key"
+ dest: /etc/pki/fedora-messaging/mediawiki{{env_suffix}}-key.pem
+ mode: 0640
+ owner: root
+ group: root
+ when: "deployment_type is defined"
+ tags:
+ - config
+
- name: startup apache
service: name=httpd enabled=yes state=started
tags:
diff --git a/roles/mediawiki/templates/LocalSettings.php.fp.j2 b/roles/mediawiki/templates/LocalSettings.php.fp.j2
index ad39df963..2ba00b8b6 100644
--- a/roles/mediawiki/templates/LocalSettings.php.fp.j2
+++ b/roles/mediawiki/templates/LocalSettings.php.fp.j2
@@ -303,6 +303,7 @@ $wgNamespacesToBeSearchedDefault = array(
NS_TEST_RESULTS_TALK => false
);
require_once "$IP/extensions/fedmsg-emit.php";
+# require_once "$IP/extensions/fedora-messaging-emit.php";
require_once "$IP/extensions/HTTP302Found/HTTP302Found.php";
require_once "$IP/extensions/RSS/RSS.php";
require_once "$IP/extensions/FedoraDocsRedirect/FedoraDocsRedirect.php";
--
2.21.0
_______________________________________________
infrastructure mailing list -- infrastructure@xxxxxxxxxxxxxxxxxxxxxxx
To unsubscribe send an email to infrastructure-leave@xxxxxxxxxxxxxxxxxxxxxxx
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://lists.fedoraproject.org/archives/list/infrastructure@xxxxxxxxxxxxxxxxxxxxxxx
