Traceback (most recent call last):
  File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 644, in cli_main
    main()
  File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 353, in main
    run_compose(
  File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 484, in run_compose
    essentials_phase.stop()
  File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 57, in stop
    self.pool.stop()
  File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop
    six.reraise(exc_info[0], exc_info[1], exc_info[2])
  File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise
    raise value
  File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run
    self.process(item, num)
  File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 79, in process
    phase.stop()
  File "/usr/lib/python3.12/site-packages/pungi/phases/base.py", line 111, in stop
    self.pool.stop()
  File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop
    six.reraise(exc_info[0], exc_info[1], exc_info[2])
  File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise
    raise value
  File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run
    self.process(item, num)
  File "/usr/lib/python3.12/site-packages/pungi/phases/ostree.py", line 68, in process
    self.worker(compose, variant, arch, config)
  File "/usr/lib/python3.12/site-packages/pungi/phases/ostree.py", line 124, in worker
    self._run_ostree_cmd(
  File "/usr/lib/python3.12/site-packages/pungi/phases/ostree.py", line 195, in _run_ostree_cmd
    runroot.run(
  File "/usr/lib/python3.12/site-packages/pungi/runroot.py", line 291, in run
    self._run_koji(
  File "/usr/lib/python3.12/site-packages/pungi/runroot.py", line 139, in _run_koji
    raise RuntimeError(
RuntimeError: Runroot task failed: 124888583. See /mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2/runroot.log for more details.
Frame _run_koji in /usr/lib/python3.12/site-packages/pungi/runroot.py at line 139
<CODE>
     132         max_retries = 3
     133         while True:
     134             output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
     135             if output["retcode"] == 0:
     136                 self._result = output
     137                 return
     138             elif attempt >= max_retries or not self._has_losetup_error(log_dir):
-->  139                 raise RuntimeError(
     140                     "Runroot task failed: %s. See %s for more details."
     141                     % (output["task_id"], log_file)
     142                 )
     143             attempt += 1
     144 
     145     def _ssh_run(self, hostname, user, command, fmt_dict=None, log_file=None):
</CODE>
<LOCALS>
                arch = 'aarch64'
             attempt = 0
             command = ['pungi-make-ostree', 'tree', '--repo=/mnt/koji/compose/iot/repo/', '--log-dir=/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2', '--treefile=/mnt/koji/compose/iot/Fedora-IoT-4
            koji_cmd = ['koji', '--profile=compose_koji', 'runroot', '--nowait', '--task-id', '--new-chroot', '--use-shell', '--channel-override=compose', '--package=pungi', '--package=ostree', '--package=rpm-ostree', '--pa
        koji_wrapper = <pungi.wrappers.kojiwrapper.KojiWrapper object at 0x7fd4f2d64bf0>
              kwargs = {'mounts': ['/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0', '/mnt/koji/compose/iot/repo/'], 'new_chroot': True, 'weight': None}
             log_dir = None
            log_file = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2/runroot.log'
         max_retries = 3
              output = {'retcode': 1, 'output': 'Watching tasks (this may be safely interrupted)...\n124888583 runroot (aarch64): free\n124888583 runroot (aarch64): free -> open (buildvm-a64-01.iad2.fedoraproject.org)\n1248
            packages = ['pungi', 'ostree', 'rpm-ostree', 'selinux-policy-targeted']
     runroot_channel = 'compose'
         runroot_tag = 'f40-build'
                self = <pungi.runroot.Runroot object at 0x7fd4f2d678f0>
        self._logger = None
        self._result = None
        self.compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
 self.runroot_method = 'koji'
</LOCALS>
Frame run in /usr/lib/python3.12/site-packages/pungi/runroot.py at line 291
<CODE>
     284             See `KojiWrapper.get_runroot_cmd()` for more information.
     285         """
     286         if self.runroot_method == "local":
     287             self._run_local(
     288                 command, log_file=log_file, packages=packages, arch=arch, **kwargs
     289             )
     290         elif self.runroot_method == "koji":
-->  291             self._run_koji(
     292                 command, log_file=log_file, packages=packages, arch=arch, **kwargs
     293             )
     294         elif self.runroot_method == "openssh":
     295             self._run_openssh(
     296                 command, log_file=log_file, packages=packages, arch=arch, **kwargs
     297             )
</CODE>
<LOCALS>
                arch = 'aarch64'
             command = ['pungi-make-ostree', 'tree', '--repo=/mnt/koji/compose/iot/repo/', '--log-dir=/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2', '--treefile=/mnt/koji/compose/iot/Fedora-IoT-4
              kwargs = {'mounts': ['/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0', '/mnt/koji/compose/iot/repo/'], 'new_chroot': True, 'weight': None}
            log_file = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2/runroot.log'
            packages = ['pungi', 'ostree', 'rpm-ostree', 'selinux-policy-targeted']
                self = <pungi.runroot.Runroot object at 0x7fd4f2d678f0>
        self._logger = None
        self._result = None
        self.compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
 self.runroot_method = 'koji'
</LOCALS>
Frame _run_ostree_cmd in /usr/lib/python3.12/site-packages/pungi/phases/ostree.py at line 195
<CODE>
     188             cmd = ["pungi-make-ostree", "tree"]
     189             for key, value in args.items():
     190                 if value is True:
     191                     cmd.append("--%s" % key)
     192                 elif value:
     193                     cmd.append("--%s=%s" % (key, value))
     194 
-->  195             runroot.run(
     196                 cmd,
     197                 log_file=log_file,
     198                 arch=arch,
     199                 packages=packages,
     200                 mounts=mounts,
     201                 new_chroot=True,
</CODE>
<LOCALS>
 additional_packages = ['selinux-policy-targeted']
                arch = 'aarch64'
                args = OrderedDict({'repo': '/mnt/koji/compose/iot/repo/', 'log-dir': '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2', 'treefile': '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/w
                 cmd = ['pungi-make-ostree', 'tree', '--repo=/mnt/koji/compose/iot/repo/', '--log-dir=/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2', '--treefile=/mnt/koji/compose/iot/Fedora-IoT-4
             compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
              config = {'version': '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN', 'force_new_commit': True, 'unified_core': True, 'treefile': 'fedora-iot.yaml', 'config_url': 'https://pagure.io/fedora-iot/ostree.git', 'conf
         config_repo = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/work/ostree-2/config_repo'
    default_packages = ['pungi', 'ostree', 'rpm-ostree']
   extra_config_file = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/work/ostree-2/extra_config.json'
                 key = 'unified-core'
            log_file = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2/runroot.log'
              mounts = ['/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0', '/mnt/koji/compose/iot/repo/']
            packages = ['pungi', 'ostree', 'rpm-ostree', 'selinux-policy-targeted']
             runroot = <pungi.runroot.Runroot object at 0x7fd4f2d678f0>
                self = <OSTreeThread(Thread-107, stopped 140551796033216)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140551796033216
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-107'
     self._native_id = 143608
       self._started = <threading.Event at 0x7fd4f2d65b20: set>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140551796033216
           self.kill = True
         self.logdir = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2'
           self.name = 'Thread-107'
      self.native_id = 143608
            self.num = 2
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4fac2eed0>
          self.repos = ['https://kojipkgs.fedoraproject.org/compose/iot/Fedora-IoT-40-20241016.0/work/$basearch/repo/f40-iot']
        self.running = False
               value = True
             variant = Variant(id="IoT", name="IoT", type="variant", parent=None)
</LOCALS>
Frame worker in /usr/lib/python3.12/site-packages/pungi/phases/ostree.py at line 124
<CODE>
     117         with open(extra_config_file, "w") as f:
     118             json.dump(new_config, f, indent=4)
     119 
     120         # Ensure target directory exists, otherwise Koji task will fail to
     121         # mount it.
     122         util.makedirs(config["ostree_repo"])
     123 
-->  124         self._run_ostree_cmd(
     125             compose, variant, arch, config, repodir, extra_config_file=extra_config_file
     126         )
     127 
     128         if compose.notifier:
     129             original_ref = get_ref_from_treefile(
     130                 os.path.join(repodir, config["treefile"]),
</CODE>
<LOCALS>
                arch = 'aarch64'
             compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
          comps_repo = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/work/$basearch/comps_repo_IoT'
              config = {'version': '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN', 'force_new_commit': True, 'unified_core': True, 'treefile': 'fedora-iot.yaml', 'config_url': 'https://pagure.io/fedora-iot/ostree.git', 'conf
   extra_config_file = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/work/ostree-2/extra_config.json'
                   f = <_io.TextIOWrapper name='/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/work/ostree-2/extra_config.json' mode='w' encoding='utf-8'>
                   k = 'update_summary'
                 msg = 'OSTree phase for variant IoT, arch aarch64'
          new_config = {'force_new_commit': True, 'unified_core': True, 'repo': [{'name': 'https:__kojipkgs.fedoraproject.org_compose_40_Fedora-40-20240414.0_compose_Everything__basearch_os_', 'baseurl': 'https://kojipkgs.f
             repodir = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/work/ostree-2/config_repo'
               repos = [{'name': 'https:__kojipkgs.fedoraproject.org_compose_40_Fedora-40-20240414.0_compose_Everything__basearch_os_', 'baseurl': 'https://kojipkgs.fedoraproject.org/compose/40/Fedora-40-20240414.0/compose/
                self = <OSTreeThread(Thread-107, stopped 140551796033216)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140551796033216
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-107'
     self._native_id = 143608
       self._started = <threading.Event at 0x7fd4f2d65b20: set>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140551796033216
           self.kill = True
         self.logdir = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2'
           self.name = 'Thread-107'
      self.native_id = 143608
            self.num = 2
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4fac2eed0>
          self.repos = ['https://kojipkgs.fedoraproject.org/compose/iot/Fedora-IoT-40-20241016.0/work/$basearch/repo/f40-iot']
        self.running = False
             variant = Variant(id="IoT", name="IoT", type="variant", parent=None)
             workdir = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/work/ostree-2'
</LOCALS>
Frame process in /usr/lib/python3.12/site-packages/pungi/phases/ostree.py at line 68
<CODE>
      61     def process(self, item, num):
      62         compose, variant, arch, config = item
      63         self.num = num
      64         failable_arches = config.get("failable", [])
      65         with util.failable(
      66             compose, util.can_arch_fail(failable_arches, arch), variant, arch, "ostree"
      67         ):
-->   68             self.worker(compose, variant, arch, config)
      69 
      70     def worker(self, compose, variant, arch, config):
      71         msg = "OSTree phase for variant %s, arch %s" % (variant.uid, arch)
      72         self.pool.log_info("[BEGIN] %s" % msg)
      73         workdir = compose.paths.work.topdir("ostree-%d" % self.num)
      74         self.logdir = compose.paths.log.topdir(
</CODE>
<LOCALS>
                arch = 'aarch64'
             compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
              config = {'version': '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN', 'force_new_commit': True, 'unified_core': True, 'treefile': 'fedora-iot.yaml', 'config_url': 'https://pagure.io/fedora-iot/ostree.git', 'conf
     failable_arches = []
                item = <ERROR WHILE CONVERTING VALUE TO STRING>
                 num = 2
                self = <OSTreeThread(Thread-107, stopped 140551796033216)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140551796033216
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-107'
     self._native_id = 143608
       self._started = <threading.Event at 0x7fd4f2d65b20: set>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140551796033216
           self.kill = True
         self.logdir = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2'
           self.name = 'Thread-107'
      self.native_id = 143608
            self.num = 2
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4fac2eed0>
          self.repos = ['https://kojipkgs.fedoraproject.org/compose/iot/Fedora-IoT-40-20241016.0/work/$basearch/repo/f40-iot']
        self.running = False
             variant = Variant(id="IoT", name="IoT", type="variant", parent=None)
</LOCALS>
Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67
<CODE>
      60 
      61             self.pool.queue_get_lock.acquire()
      62             self.pool.queue_processed += 1
      63             num = self.pool.queue_processed
      64             self.pool.queue_get_lock.release()
      65 
      66             try:
-->   67                 self.process(item, num)
      68             except:
      69                 self.failed = True
      70                 self.pool.exceptions.append(sys.exc_info())
      71                 self.pool.kill()
      72 
      73     def process(self, item, num):
</CODE>
<LOCALS>
                item = <ERROR WHILE CONVERTING VALUE TO STRING>
                 num = 2
                self = <OSTreeThread(Thread-107, stopped 140551796033216)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140551796033216
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-107'
     self._native_id = 143608
       self._started = <threading.Event at 0x7fd4f2d65b20: set>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140551796033216
           self.kill = True
         self.logdir = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2'
           self.name = 'Thread-107'
      self.native_id = 143608
            self.num = 2
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4fac2eed0>
          self.repos = ['https://kojipkgs.fedoraproject.org/compose/iot/Fedora-IoT-40-20241016.0/work/$basearch/repo/f40-iot']
        self.running = False
</LOCALS>
Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719
<CODE>
     712 
     713     def reraise(tp, value, tb=None):
     714         try:
     715             if value is None:
     716                 value = tp()
     717             if value.__traceback__ is not tb:
     718                 raise value.with_traceback(tb)
-->  719             raise value
     720         finally:
     721             value = None
     722             tb = None
     723 
     724 else:
     725     def exec_(_code_, _globs_=None, _locs_=None):
</CODE>
<LOCALS>
                  tb = None
                  tp = <class 'RuntimeError'>
               value = None
</LOCALS>
Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120
<CODE>
     113         """
     114         for i in self.threads:
     115             i.running = False
     116         for i in self.threads:
     117             i.join()
     118         if self.exceptions:
     119             exc_info = self.exceptions[0]
-->  120             six.reraise(exc_info[0], exc_info[1], exc_info[2])
     121 
     122     def kill(self):
     123         for i in self.threads:
     124             i.kill = True
     125             i.running = False
     126 
</CODE>
<LOCALS>
            exc_info = <ERROR WHILE CONVERTING VALUE TO STRING>
                   i = <OSTreeThread(Thread-107, stopped 140551796033216)>
                self = <kobo.threads.ThreadPool object at 0x7fd4fac2eed0>
        self._logger = <Logger pungi (DEBUG)>
     self.exceptions = [(<class 'RuntimeError'>, RuntimeError('Runroot task failed: 124888583. See /mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2/runroot.log for more details.'), <traceback object 
          self.queue = <queue.Queue object at 0x7fd4fa9f78c0>
 self.queue_get_lock = <unlocked _thread.lock object at 0x7fd4fac084c0>
self.queue_processed = 2
 self.queue_put_lock = <unlocked _thread.lock object at 0x7fd4fa9b6e00>
    self.queue_total = 2
        self.threads = [<OSTreeThread(Thread-106, stopped 140551785547456)>, <OSTreeThread(Thread-107, stopped 140551796033216)>]
</LOCALS>
Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/base.py at line 111
<CODE>
     104                 "Note that variants can be excluded in configuration file"
     105             )
     106 
     107     def stop(self):
     108         if self.finished:
     109             return
     110         if hasattr(self, "pool"):
-->  111             self.pool.stop()
     112         self.finished = True
     113         self.compose.log_info("[DONE ] %s" % self.msg)
     114 
     115         if hasattr(self, "_start_time"):
     116             self.compose.log_info(
     117                 "PHASE %s took %d seconds"
</CODE>
<LOCALS>
                self = <pungi.phases.ostree.OSTreePhase object at 0x7fd4faa2d130>
       self._skipped = False
    self._start_time = 1729085942.863663
        self.compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
       self.finished = False
            self.msg = '---------- PHASE: OSTREE ----------'
           self.name = 'ostree'
   self.pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7fd4fac2eea0>
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4fac2eed0>
  self.used_patterns = {'^IoT$'}
</LOCALS>
Frame process in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 79
<CODE>
      72             self.pool.queue_total,
      73             phases_names,
      74         )
      75         self.pool.log_info("[BEGIN] %s" % (msg,))
      76 
      77         for phase in pipeline:
      78             phase.start()
-->   79             phase.stop()
      80 
      81         self.pool.log_info("[DONE ] %s" % (msg,))
</CODE>
<LOCALS>
                item = [<pungi.phases.ostree.OSTreePhase object at 0x7fd4faa2d130>, <pungi.phases.ostree_installer.OstreeInstallerPhase object at 0x7fd4faa2ce60>]
                 msg = 'Running pipeline (4/5). Phases: ostree, ostree_installer'
                 num = 4
               phase = <pungi.phases.ostree.OSTreePhase object at 0x7fd4faa2d130>
        phases_names = 'ostree, ostree_installer'
            pipeline = [<pungi.phases.ostree.OSTreePhase object at 0x7fd4faa2d130>, <pungi.phases.ostree_installer.OstreeInstallerPhase object at 0x7fd4faa2ce60>]
                self = <PipelineThread(Thread-95, stopped 140551754090176)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140551754090176
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-95'
     self._native_id = 143583
       self._started = <threading.Event at 0x7fd4f2d59070: set>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140551754090176
           self.kill = True
           self.name = 'Thread-95'
      self.native_id = 143583
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4f2e43200>
        self.running = False
</LOCALS>
Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67
<CODE>
      60 
      61             self.pool.queue_get_lock.acquire()
      62             self.pool.queue_processed += 1
      63             num = self.pool.queue_processed
      64             self.pool.queue_get_lock.release()
      65 
      66             try:
-->   67                 self.process(item, num)
      68             except:
      69                 self.failed = True
      70                 self.pool.exceptions.append(sys.exc_info())
      71                 self.pool.kill()
      72 
      73     def process(self, item, num):
</CODE>
<LOCALS>
                item = [<pungi.phases.ostree.OSTreePhase object at 0x7fd4faa2d130>, <pungi.phases.ostree_installer.OstreeInstallerPhase object at 0x7fd4faa2ce60>]
                 num = 4
                self = <PipelineThread(Thread-95, stopped 140551754090176)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140551754090176
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-95'
     self._native_id = 143583
       self._started = <threading.Event at 0x7fd4f2d59070: set>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140551754090176
           self.kill = True
           self.name = 'Thread-95'
      self.native_id = 143583
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4f2e43200>
        self.running = False
</LOCALS>
Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719
<CODE>
     712 
     713     def reraise(tp, value, tb=None):
     714         try:
     715             if value is None:
     716                 value = tp()
     717             if value.__traceback__ is not tb:
     718                 raise value.with_traceback(tb)
-->  719             raise value
     720         finally:
     721             value = None
     722             tb = None
     723 
     724 else:
     725     def exec_(_code_, _globs_=None, _locs_=None):
</CODE>
<LOCALS>
                  tb = None
                  tp = <class 'RuntimeError'>
               value = None
</LOCALS>
Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120
<CODE>
     113         """
     114         for i in self.threads:
     115             i.running = False
     116         for i in self.threads:
     117             i.join()
     118         if self.exceptions:
     119             exc_info = self.exceptions[0]
-->  120             six.reraise(exc_info[0], exc_info[1], exc_info[2])
     121 
     122     def kill(self):
     123         for i in self.threads:
     124             i.kill = True
     125             i.running = False
     126 
</CODE>
<LOCALS>
            exc_info = <ERROR WHILE CONVERTING VALUE TO STRING>
                   i = <PipelineThread(Thread-96, stopped 140552169326272)>
                self = <kobo.threads.ThreadPool object at 0x7fd4f2e43200>
        self._logger = <Logger pungi (DEBUG)>
     self.exceptions = [(<class 'RuntimeError'>, RuntimeError('Runroot task failed: 124888583. See /mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2/runroot.log for more details.'), <traceback object 
          self.queue = <queue.Queue object at 0x7fd4fac2f650>
 self.queue_get_lock = <unlocked _thread.lock object at 0x7fd4f2d48180>
self.queue_processed = 5
 self.queue_put_lock = <unlocked _thread.lock object at 0x7fd4f2d483c0>
    self.queue_total = 5
        self.threads = [<PipelineThread(Thread-92, stopped 140551796033216)>, <PipelineThread(Thread-93, stopped 140551944931008)>, <PipelineThread(Thread-94, stopped 140551785547456)>, <PipelineThread(Thread-95, stopped 14
</LOCALS>
Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 57
<CODE>
      50 
      51         self.pool.start()
      52 
      53     def stop(self):
      54         if self.finished:
      55             return
      56         if hasattr(self, "pool"):
-->   57             self.pool.stop()
      58         self.finished = True
      59         self.compose.log_info("[DONE ] %s" % self.msg)
      60 
      61 
      62 class PipelineThread(WorkerThread):
      63     """
</CODE>
<LOCALS>
                self = <pungi.phases.weaver.WeaverPhase object at 0x7fd4fa9f7bf0>
 self._phases_schema = <ERROR WHILE CONVERTING VALUE TO STRING>
        self.compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
       self.finished = False
            self.msg = '---------- PHASE: WEAVER ----------'
           self.name = 'weaver'
           self.pool = <kobo.threads.ThreadPool object at 0x7fd4f2e43200>
</LOCALS>
Frame run_compose in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 484
<CODE>
     477         (gather_phase, createrepo_phase),
     478         extrafiles_phase,
     479         (ostree_phase, ostree_installer_phase),
     480         ostree_container_phase,
     481     )
     482     essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema)
     483     essentials_phase.start()
-->  484     essentials_phase.stop()
     485 
     486     # write treeinfo before ISOs are created
     487     for variant in compose.get_variants():
     488         for arch in variant.arches + ["src"]:
     489             pungi.metadata.write_tree_info(
     490                 compose, arch, variant, bi=buildinstall_phase
</CODE>
<LOCALS>
  buildinstall_phase = <pungi.phases.buildinstall.BuildinstallPhase object at 0x7fd4fa9f78f0>
             compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
    config_copy_path = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/global/config-copy'
    config_dump_full = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/global/config-dump.global.log'
         config_file = '/root/pungi-iot/fedora-iot.conf'
  create_latest_link = True
     createiso_phase = <pungi.phases.createiso.CreateisoPhase object at 0x7fd4faa2d310>
    createrepo_phase = <pungi.phases.createrepo.CreaterepoPhase object at 0x7fd4faa2cbf0>
              errors = []
    essentials_phase = <pungi.phases.weaver.WeaverPhase object at 0x7fd4fa9f7bf0>
   essentials_schema = <ERROR WHILE CONVERTING VALUE TO STRING>
    extra_isos_phase = <pungi.phases.extra_isos.ExtraIsosPhase object at 0x7fd4faa2d4f0>
    extrafiles_phase = <pungi.phases.extra_files.ExtraFilesPhase object at 0x7fd4faa2d550>
                   f = <_io.TextIOWrapper name='/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/global/config-dump.global.log' mode='w' encoding='utf-8'>
        gather_phase = <pungi.phases.gather.GatherPhase object at 0x7fd4fac01040>
   image_build_phase = <pungi.phases.image_build.ImageBuildPhase object at 0x7fd4faa2db50>
image_checksum_phase = <pungi.phases.image_checksum.ImageChecksumPhase object at 0x7fd4fac2ed20>
image_container_phase = <pungi.phases.image_container.ImageContainerPhase object at 0x7fd4faa2eab0>
          init_phase = <pungi.phases.init.InitPhase object at 0x7fd4faa2d2b0>
  installed_pkgs_log = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/global/installed-pkgs.global.log'
     kiwibuild_phase = <pungi.phases.kiwibuild.KiwiBuildPhase object at 0x7fd4fc0e75f0>
latest_link_components = -1
  latest_link_status = None
     livemedia_phase = <pungi.phases.livemedia_phase.LiveMediaPhase object at 0x7fd4fa9f7a70>
          osbs_phase = <pungi.phases.osbs.OSBSPhase object at 0x7fd4faa2e660>
       osbuild_phase = <pungi.phases.osbuild.OSBuildPhase object at 0x7fd4faa2e210>
ostree_container_phase = <pungi.phases.ostree_container.OSTreeContainerPhase object at 0x7fd4faa2d100>
ostree_installer_phase = <pungi.phases.ostree_installer.OstreeInstallerPhase object at 0x7fd4faa2ce60>
        ostree_phase = <pungi.phases.ostree.OSTreePhase object at 0x7fd4faa2d130>
               phase = <pungi.phases.kiwibuild.KiwiBuildPhase object at 0x7fd4fc0e75f0>
        pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7fd4fac2eea0>
               pungi = <module 'pungi' from '/usr/lib/python3.12/site-packages/pungi/__init__.py'>
   repoclosure_phase = <pungi.phases.repoclosure.RepoclosurePhase object at 0x7fd4fa9e0410>
          test_phase = <pungi.phases.test.TestPhase object at 0x7fd4faa2eed0>
</LOCALS>
Frame main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 353
<CODE>
     346             logger.error("Config validation failed with the error: %s" % error)
     347         fail_to_start("Config validation failed", errors=errors)
     348         sys.exit(1)
     349 
     350     notifier.compose = compose
     351     COMPOSE = compose
     352     try:
-->  353         run_compose(
     354             compose,
     355             create_latest_link=create_latest_link,
     356             latest_link_status=latest_link_status,
     357             latest_link_components=latest_link_components,
     358         )
     359     except UnsignedPackagesError:
</CODE>
<LOCALS>
             Compose = <class 'pungi.compose.Compose'>
PHASES_NAMES_MODIFIED = ['buildinstall', 'createiso', 'createrepo', 'extra_files', 'extra_isos', 'gather', 'image_build', 'image_checksum', 'image_container', 'init', 'kiwibuild', 'live_media', 'osbs', 'osbuild', 'ostree_con
               abort = <function main.<locals>.abort at 0x7fd4fad23920>
             compose = <pungi.compose.Compose object at 0x7fd4fac02c30>
         compose_dir = '/mnt/koji/compose/iot/Fedora-IoT-40-20241016.0'
        compose_type = 'production'
                conf = {'release_name': 'Fedora-IoT', 'release_short': 'Fedora-IoT', 'release_version': '40', 'skip_phases': ['buildinstall', 'createrepo', 'extra_files'], 'comps_file': {'scm': 'git', 'repo': 'https://pagur
  create_latest_link = True
              errors = []
       fail_to_start = <function main.<locals>.fail_to_start at 0x7fd4fad237e0>
               group = <argparse._MutuallyExclusiveGroup object at 0x7fd4facf23c0>
                kobo = <module 'kobo' from '/usr/lib/python3.12/site-packages/kobo/__init__.py'>
latest_link_components = -1
  latest_link_status = None
              logger = <Logger pungi (DEBUG)>
            notifier = <pungi.notifier.PungiNotifier object at 0x7fd4fd1905f0>
                opts = Namespace(target_dir='/mnt/koji/compose/iot', compose_dir=None, label='RC-20241016.0', no_label=False, supported=False, old_composes=['/mnt/koji/compose/iot'], config='/root/pungi-iot/fedora-iot.conf'
              parser = ArgumentParser(prog='pungi-koji', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)
           productmd = <module 'productmd' from '/usr/lib/python3.12/site-packages/productmd/__init__.py'>
               pungi = <module 'pungi' from '/usr/lib/python3.12/site-packages/pungi/__init__.py'>
                  rv = None
            warnings = []
</LOCALS>
Frame cli_main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 644
<CODE>
     637 
     638 
     639 def cli_main():
     640     signal.signal(signal.SIGINT, sigterm_handler)
     641     signal.signal(signal.SIGTERM, sigterm_handler)
     642 
     643     try:
-->  644         main()
     645     except (Exception, KeyboardInterrupt) as ex:
     646         if COMPOSE:
     647             COMPOSE.log_error("Compose run failed: %s" % ex)
     648             COMPOSE.traceback(show_locals=getattr(ex, "show_locals", True))
     649             COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
     650             COMPOSE.write_status("DOOMED")
</CODE>
<LOCALS>
                  ex = RuntimeError('Runroot task failed: 124888583. See /mnt/koji/compose/iot/Fedora-IoT-40-20241016.0/logs/aarch64/IoT/ostree-2/runroot.log for more details.')
</LOCALS>