From b8ea345c48668dc15d35a6ef4ad644099bc9e67d Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 17 2023 13:08:50 +0000
Subject: [PATCH 1/3] vm: Retry libvirt connection


Related: https://pagure.io/koji/issue/985

---

diff --git a/vm/kojivmd b/vm/kojivmd
index 0853bf5..be87e66 100755
--- a/vm/kojivmd
+++ b/vm/kojivmd
@@ -59,6 +59,40 @@ from koji.tasks import (  # noqa: F401
 )
 
 
+class LibvirtCallable(object):
+    # Wrapper class for libvirt call, used in LibvirtConnection
+    def __init__(self, libvirt_conn, method):
+        self.conn = libvirt_conn
+        self.method = method
+
+    def __call__(self, *args, **kwargs):
+        max_retries = 3
+        retry = 0
+        for retry in range(max_retries):
+            try:
+                return getattr(self.conn._conn, self.method)(*args, **kwargs)
+            except libvirt.libvirtError as ex:
+                if ex.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR and retry < max_retries:
+                    logging.getLogger("koji.vm").warning(
+                        'Internal libvirt error: %s, retry #%d/%d' % (ex, retry, max_retries))
+                    time.sleep(5)
+                    self.conn._conn = libvirt.open(None)
+                else:
+                    raise
+
+
+class LibvirtConnection(object):
+    # Wrapper class for libvirt connection retrying calls failed on libvirt connection
+    # e.g. libvirtd restart. Without it kojivmd stucks in main loop
+    def __init__(self) -> None:
+        self._conn = libvirt.open(None)
+
+    def __getattr__(self, name):
+        if name == '_conn':
+            return self._conn
+        return LibvirtCallable(self, name)
+
+
 # Register libvirt handler
 def libvirt_callback(ignore, err):
     if err[3] != libvirt.VIR_ERR_ERROR:
@@ -866,7 +900,7 @@ class VMExecTask(BaseTaskHandler):
 
         self.task_info = task_info
 
-        conn = libvirt.open(None)
+        conn = LibvirtConnection()
         clone_name = self.clone(conn, name, opts)
         self.logger.debug('Cloned VM %s to %s', name, clone_name)
         try:
@@ -916,7 +950,7 @@ class VMExecTask(BaseTaskHandler):
 class VMTaskManager(TaskManager):
     def __init__(self, options, session):
         super(VMTaskManager, self).__init__(options, session)
-        self.libvirt_conn = libvirt.open(None)
+        self.libvirt_conn = LibvirtConnection()
         self.macaddrs = {}
         self.macaddr_lock = threading.Lock()
         self.expired_vms = {}

From 971110243b105c74e2b36b3dd96a945ae50b98ad Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 20 2023 10:48:24 +0000
Subject: [PATCH 2/3] try to close failing connection


---

diff --git a/vm/kojivmd b/vm/kojivmd
index be87e66..fac64ac 100755
--- a/vm/kojivmd
+++ b/vm/kojivmd
@@ -76,6 +76,10 @@ class LibvirtCallable(object):
                     logging.getLogger("koji.vm").warning(
                         'Internal libvirt error: %s, retry #%d/%d' % (ex, retry, max_retries))
                     time.sleep(5)
+                    try:
+                        self.conn._conn.close()
+                    except Exception:
+                        pass
                     self.conn._conn = libvirt.open(None)
                 else:
                     raise

From f5ee0b8bce8e28cf5c0ed6a8ed4dfd2480d5e113 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 22 2023 13:37:55 +0000
Subject: [PATCH 3/3] use wrapper only in main thread


---

diff --git a/vm/kojivmd b/vm/kojivmd
index fac64ac..ae7bff2 100755
--- a/vm/kojivmd
+++ b/vm/kojivmd
@@ -904,7 +904,7 @@ class VMExecTask(BaseTaskHandler):
 
         self.task_info = task_info
 
-        conn = LibvirtConnection()
+        conn = libvirt.open(None)
         clone_name = self.clone(conn, name, opts)
         self.logger.debug('Cloned VM %s to %s', name, clone_name)
         try: