Traceback (most recent call last): File "/usr/lib/python3.10/site-packages/pungi/scripts/pungi_koji.py", line 670, in cli_main main() File "/usr/lib/python3.10/site-packages/pungi/scripts/pungi_koji.py", line 347, in main run_compose( File "/usr/lib/python3.10/site-packages/pungi/scripts/pungi_koji.py", line 549, in run_compose extra_phase.stop() File "/usr/lib/python3.10/site-packages/pungi/phases/weaver.py", line 57, in stop self.pool.stop() File "/usr/lib/python3.10/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.10/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.10/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.10/site-packages/pungi/phases/weaver.py", line 79, in process phase.stop() File "/usr/lib/python3.10/site-packages/pungi/phases/weaver.py", line 57, in stop self.pool.stop() File "/usr/lib/python3.10/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.10/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.10/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.10/site-packages/pungi/phases/weaver.py", line 79, in process phase.stop() File "/usr/lib/python3.10/site-packages/pungi/phases/base.py", line 111, in stop self.pool.stop() File "/usr/lib/python3.10/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.10/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.10/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.10/site-packages/pungi/phases/image_build.py", line 218, in process self.worker(num, compose, variant, subvariant, cmd, buildinstall_phase) File "/usr/lib/python3.10/site-packages/pungi/phases/image_build.py", line 287, in worker raise RuntimeError( RuntimeError: ImageBuild task failed: 93063254. See /mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/aarch64-armhfp-x86_64/imagebuild-IoT-IoT-raw-xz.aarch64-armhfp-x86_64.log for more details. Frame worker in /usr/lib/python3.10/site-packages/pungi/phases/image_build.py at line 287 280 # [workaround] Increased time delay from 3 to 10 sec until the issue in 281 # koji gets fixed https://pagure.io/koji/issue/2138 282 time.sleep(num * 10) 283 output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file) 284 self.pool.log_debug("build-image outputs: %s" % (output)) 285 if output["retcode"] != 0: 286 self.fail(compose, cmd) --> 287 raise RuntimeError( 288 "ImageBuild task failed: %s. See %s for more details." 289 % (output["task_id"], log_file) 290 ) 291 292 # copy image to images/ 293 image_infos = [] arches = ['aarch64', 'armhfp', 'x86_64'] buildinstall_phase = cmd = {'original_image_conf': {'image-build': {'format': [('raw-xz', 'raw.xz')], 'name': 'Fedora-IoT', 'kickstart': 'fedora-iot.ks', 'distro': 'Fedora-30', 'disk_size': 4, 'arches': ['armhfp', 'aarch64', 'x compose = dash_arches = 'aarch64-armhfp-x86_64' external_repo_checksum = None formats = 'raw-xz' koji_cmd = ['koji', '--profile=compose_koji', 'image-build', '--config=/mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/work/image-build/IoT/raw-xz_Fedora-IoT_aarch64-armhfp-x86_64.cfg', '--wait'] koji_wrapper = log_file = '/mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/aarch64-armhfp-x86_64/imagebuild-IoT-IoT-raw-xz.aarch64-armhfp-x86_64.log' metadata_file = '/mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/aarch64-armhfp-x86_64/imagebuild-IoT-IoT-raw-xz.aarch64-armhfp-x86_64.reuse.json' msg = 'Creating image (formats: raw-xz, arches: aarch64-armhfp-x86_64, variant: IoT, subvariant: IoT)' num = 1 output = {'retcode': 1, 'output': "Created task: 93063254\nTask info: https://koji.fedoraproject.org/koji/taskinfo?taskID=93063254\nWatching tasks (this may be safely interrupted)...\n93063254 image (['aarch64 self = self._args = self._daemonic = False self._ident = 139840234448448 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-147' self._native_id = 14166 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.can_fail = False self.daemon = False self.failable_arches = '' self.failed = True self.get_timeout = 1 self.ident = 139840234448448 self.kill = True self.name = 'Thread-147' self.native_id = 14166 self.pool = self.running = False subvariant = 'IoT' variant = Variant(id="IoT", name="IoT", type="variant", parent=None) Frame process in /usr/lib/python3.10/site-packages/pungi/phases/image_build.py at line 218 211 self.can_fail, 212 variant, 213 "*", 214 "image-build", 215 subvariant, 216 logger=self.pool._logger, 217 ): --> 218 self.worker(num, compose, variant, subvariant, cmd, buildinstall_phase) 219 220 def worker(self, num, compose, variant, subvariant, cmd, buildinstall_phase): 221 arches = cmd["image_conf"]["image-build"]["arches"] 222 formats = "-".join(cmd["image_conf"]["image-build"]["format"]) 223 dash_arches = "-".join(arches) 224 log_file = compose.paths.log.log_file( buildinstall_phase = cmd = {'original_image_conf': {'image-build': {'format': [('raw-xz', 'raw.xz')], 'name': 'Fedora-IoT', 'kickstart': 'fedora-iot.ks', 'distro': 'Fedora-30', 'disk_size': 4, 'arches': ['armhfp', 'aarch64', 'x compose = item = num = 1 self = self._args = self._daemonic = False self._ident = 139840234448448 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-147' self._native_id = 14166 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.can_fail = False self.daemon = False self.failable_arches = '' self.failed = True self.get_timeout = 1 self.ident = 139840234448448 self.kill = True self.name = 'Thread-147' self.native_id = 14166 self.pool = self.running = False subvariant = 'IoT' variant = Variant(id="IoT", name="IoT", type="variant", parent=None) Frame run in /usr/lib/python3.10/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = num = 1 self = self._args = self._daemonic = False self._ident = 139840234448448 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-147' self._native_id = 14166 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.can_fail = False self.daemon = False self.failable_arches = '' self.failed = True self.get_timeout = 1 self.ident = 139840234448448 self.kill = True self.name = 'Thread-147' self.native_id = 14166 self.pool = self.running = False Frame reraise in /usr/lib/python3.10/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.10/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('ImageBuild task failed: 93063254. See /mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/aarch64-armhfp-x86_64/imagebuild-IoT-IoT-raw-xz.aarch64-armhfp-x86_64. self.queue = self.queue_get_lock = self.queue_processed = 1 self.queue_put_lock = self.queue_total = 1 self.threads = [] Frame stop in /usr/lib/python3.10/site-packages/pungi/phases/base.py at line 111 104 "Note that variants can be excluded in configuration file" 105 ) 106 107 def stop(self): 108 if self.finished: 109 return 110 if hasattr(self, "pool"): --> 111 self.pool.stop() 112 self.finished = True 113 self.compose.log_info("[DONE ] %s" % self.msg) 114 115 if hasattr(self, "_start_time"): 116 self.compose.log_info( 117 "PHASE %s took %d seconds" self = self._skipped = False self._start_time = 1665859230.7091792 self.buildinstall_phase = self.compose = self.finished = False self.logger = self.msg = '---------- PHASE: IMAGE_BUILD ----------' self.name = 'image_build' self.pool = self.used_patterns = {'^IoT$'} Frame process in /usr/lib/python3.10/site-packages/pungi/phases/weaver.py at line 79 72 self.pool.queue_total, 73 phases_names, 74 ) 75 self.pool.log_info("[BEGIN] %s" % (msg,)) 76 77 for phase in pipeline: 78 phase.start() --> 79 phase.stop() 80 81 self.pool.log_info("[DONE ] %s" % (msg,)) item = [] msg = 'Running pipeline (4/6). Phases: image_build' num = 4 phase = phases_names = 'image_build' pipeline = [] self = self._args = self._daemonic = False self._ident = 139840242841152 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-138' self._native_id = 14134 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 139840242841152 self.kill = True self.name = 'Thread-138' self.native_id = 14134 self.pool = self.running = False Frame run in /usr/lib/python3.10/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = [] num = 4 self = self._args = self._daemonic = False self._ident = 139840242841152 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-138' self._native_id = 14134 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 139840242841152 self.kill = True self.name = 'Thread-138' self.native_id = 14134 self.pool = self.running = False Frame reraise in /usr/lib/python3.10/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.10/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('ImageBuild task failed: 93063254. See /mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/aarch64-armhfp-x86_64/imagebuild-IoT-IoT-raw-xz.aarch64-armhfp-x86_64. self.queue = self.queue_get_lock = self.queue_processed = 6 self.queue_put_lock = self.queue_total = 6 self.threads = [, , , Frame stop in /usr/lib/python3.10/site-packages/pungi/phases/weaver.py at line 57 50 51 self.pool.start() 52 53 def stop(self): 54 if self.finished: 55 return 56 if hasattr(self, "pool"): --> 57 self.pool.stop() 58 self.finished = True 59 self.compose.log_info("[DONE ] %s" % self.msg) 60 61 62 class PipelineThread(WorkerThread): 63 """ self = self._phases_schema = self.compose = self.finished = False self.msg = '---------- PHASE: WEAVER ----------' self.name = 'weaver' self.pool = Frame process in /usr/lib/python3.10/site-packages/pungi/phases/weaver.py at line 79 72 self.pool.queue_total, 73 phases_names, 74 ) 75 self.pool.log_info("[BEGIN] %s" % (msg,)) 76 77 for phase in pipeline: 78 phase.start() --> 79 phase.stop() 80 81 self.pool.log_info("[DONE ] %s" % (msg,)) item = [, ] msg = 'Running pipeline (1/3). Phases: weaver, weaver' num = 1 phase = phases_names = 'weaver, weaver' pipeline = [, ] self = self._args = self._daemonic = False self._ident = 139840217663040 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-132' self._native_id = 14128 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 139840217663040 self.kill = True self.name = 'Thread-132' self.native_id = 14128 self.pool = self.running = False Frame run in /usr/lib/python3.10/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = [, ] num = 1 self = self._args = self._daemonic = False self._ident = 139840217663040 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-132' self._native_id = 14128 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 139840217663040 self.kill = True self.name = 'Thread-132' self.native_id = 14128 self.pool = self.running = False Frame reraise in /usr/lib/python3.10/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.10/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('ImageBuild task failed: 93063254. See /mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/aarch64-armhfp-x86_64/imagebuild-IoT-IoT-raw-xz.aarch64-armhfp-x86_64. self.queue = self.queue_get_lock = self.queue_processed = 3 self.queue_put_lock = self.queue_total = 3 self.threads = [, , ] Frame stop in /usr/lib/python3.10/site-packages/pungi/phases/weaver.py at line 57 50 51 self.pool.start() 52 53 def stop(self): 54 if self.finished: 55 return 56 if hasattr(self, "pool"): --> 57 self.pool.stop() 58 self.finished = True 59 self.compose.log_info("[DONE ] %s" % self.msg) 60 61 62 class PipelineThread(WorkerThread): 63 """ self = self._phases_schema = self.compose = self.finished = False self.msg = '---------- PHASE: WEAVER ----------' self.name = 'weaver' self.pool = Frame run_compose in /usr/lib/python3.10/site-packages/pungi/scripts/pungi_koji.py at line 549 542 (compose_images_phase, post_image_phase), 543 osbs_phase, 544 repoclosure_phase, 545 ) 546 extra_phase = pungi.phases.WeaverPhase(compose, extra_phase_schema) 547 548 extra_phase.start() --> 549 extra_phase.stop() 550 551 pungi.metadata.write_compose_info(compose) 552 if not ( 553 buildinstall_phase.skip() 554 and ostree_installer_phase.skip() 555 and createiso_phase.skip() arch = 'src' buildinstall_phase = compose = compose_images_phase = compose_images_schema = config_copy_path = '/mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/global/config-copy' config_dump_full = '/mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/global/config-dump.global.log' config_file = '/root/pungi-iot/fedora-iot.conf' create_latest_link = True createiso_phase = createrepo_phase = errors = [] essentials_phase = essentials_schema = extra_isos_phase = extra_phase = extra_phase_schema = extrafiles_phase = f = <_io.TextIOWrapper name='/mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/global/config-dump.global.log' mode='w' encoding='UTF-8'> gather_phase = image_build_phase = image_checksum_phase = image_container_phase = init_phase = latest_link_components = -1 latest_link_status = None liveimages_phase = livemedia_phase = osbs_phase = osbuild_phase = ostree_installer_phase = ostree_phase = phase = pkgset_phase = post_image_phase = pungi = repoclosure_phase = test_phase = timestamp = '1665859230.701287' variant = Variant(id="IoT", name="IoT", type="variant", parent=None) Frame main in /usr/lib/python3.10/site-packages/pungi/scripts/pungi_koji.py at line 347 340 logger.error("Config validation failed with the error: %s" % error) 341 fail_to_start("Config validation failed", errors=errors) 342 sys.exit(1) 343 344 notifier.compose = compose 345 COMPOSE = compose 346 try: --> 347 run_compose( 348 compose, 349 create_latest_link=create_latest_link, 350 latest_link_status=latest_link_status, 351 latest_link_components=latest_link_components, 352 ) 353 except UnsignedPackagesError: Compose = PHASES_NAMES_MODIFIED = ['buildinstall', 'createiso', 'createrepo', 'extra_files', 'extra_isos', 'gather', 'image_build', 'image_checksum', 'image_container', 'init', 'live_images', 'live_media', 'osbs', 'osbuild', 'ostree', abort = .abort at 0x7f2f380451b0> compose = compose_dir = '/mnt/koji/compose/iot/Fedora-IoT-36-20221015.0' compose_type = 'production' conf = {'release_name': 'Fedora-IoT', 'release_short': 'Fedora-IoT', 'release_version': '36', 'skip_phases': ['buildinstall', 'createrepo', 'extra_files'], 'ostree_installer_overwrite': True, 'comps_file': { create_latest_link = True errors = [] fail_to_start = .fail_to_start at 0x7f2f38045090> group = kobo = latest_link_components = -1 latest_link_status = None logger = notifier = opts = Namespace(target_dir='/mnt/koji/compose/iot', compose_dir=None, label='RC-20221015.0', no_label=False, supported=False, old_composes=['/mnt/koji/compose/iot'], config='/root/pungi-iot/fedora-iot.conf' parser = ArgumentParser(prog='pungi-koji', usage=None, description=None, formatter_class=, conflict_handler='error', add_help=True) productmd = pungi = rv = None warnings = [] Frame cli_main in /usr/lib/python3.10/site-packages/pungi/scripts/pungi_koji.py at line 670 663 664 665 def cli_main(): 666 signal.signal(signal.SIGINT, sigterm_handler) 667 signal.signal(signal.SIGTERM, sigterm_handler) 668 669 try: --> 670 main() 671 except (Exception, KeyboardInterrupt) as ex: 672 if COMPOSE: 673 COMPOSE.log_error("Compose run failed: %s" % ex) 674 COMPOSE.traceback() 675 COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir) 676 COMPOSE.write_status("DOOMED") ex = RuntimeError('ImageBuild task failed: 93063254. See /mnt/koji/compose/iot/Fedora-IoT-36-20221015.0/logs/aarch64-armhfp-x86_64/imagebuild-IoT-IoT-raw-xz.aarch64-armhfp-x86_64.log for more details.')