|
2025-10-15
ยง
|
| 09:16 |
<klausman@deploy2002> |
helmfile [ml-serve-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 09:14 |
<klausman@deploy2002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 09:13 |
<klausman@deploy2002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 09:01 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2046.codfw.wmnet'] |
[production] |
| 09:01 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2046.codfw.wmnet'] |
[production] |
| 08:59 |
<Amir1> |
mwscript-k8s -- purgeUserOptions.php --wiki=loginwiki (T406724) |
[production] |
| 08:57 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2047.codfw.wmnet'] |
[production] |
| 08:51 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2047.codfw.wmnet'] |
[production] |
| 08:49 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2048.codfw.wmnet'] |
[production] |
| 08:47 |
<marostegui@cumin1003> |
START - Cookbook sre.mysql.pool db2206 gradually with 4 steps - Pool db2206.codfw.wmnet in after cloning |
[production] |
| 08:44 |
<mvernon@cumin1002> |
END (PASS) - Cookbook sre.swift.roll-restart-reboot-swift-thanos-proxies (exit_code=0) rolling reboot on A:thanos-fe |
[production] |
| 08:41 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2048.codfw.wmnet'] |
[production] |
| 08:41 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2049.codfw.wmnet'] |
[production] |
| 08:33 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1057 (re)pooling @ 100%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83918 and previous config saved to /var/cache/conftool/dbconfig/20251015-083339-root.json |
[production] |
| 08:33 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1052 (re)pooling @ 100%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83917 and previous config saved to /var/cache/conftool/dbconfig/20251015-083333-root.json |
[production] |
| 08:30 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2049.codfw.wmnet'] |
[production] |
| 08:29 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2050.codfw.wmnet'] |
[production] |
| 08:22 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2050.codfw.wmnet'] |
[production] |
| 08:22 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2051.codfw.wmnet'] |
[production] |
| 08:19 |
<hashar@deploy2002> |
rebuilt and synchronized wikiversions files: group1 to 1.45.0-wmf.23 refs T405679 |
[production] |
| 08:18 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1057 (re)pooling @ 75%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83916 and previous config saved to /var/cache/conftool/dbconfig/20251015-081833-root.json |
[production] |
| 08:18 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1052 (re)pooling @ 75%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83915 and previous config saved to /var/cache/conftool/dbconfig/20251015-081827-root.json |
[production] |
| 08:14 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2051.codfw.wmnet'] |
[production] |
| 08:14 |
<elukey@cumin2002> |
END (ERROR) - Cookbook sre.hardware.upgrade-firmware (exit_code=97) upgrade firmware for hosts ['cp2052.codfw.wmnet'] |
[production] |
| 08:14 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2052.codfw.wmnet'] |
[production] |
| 08:13 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2052.codfw.wmnet'] |
[production] |
| 08:13 |
<fceratto@cumin1002> |
END (PASS) - Cookbook sre.mysql.clone_es (exit_code=0) of es2032.codfw.wmnet onto es2053.codfw.wmnet |
[production] |
| 08:13 |
<fceratto@cumin1002> |
END (PASS) - Cookbook sre.mysql.pool (exit_code=0) es2032 gradually with 4 steps - Pool es2032.codfw.wmnet in after cloning |
[production] |
| 08:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host urldownloader1004.wikimedia.org |
[production] |
| 08:09 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host urldownloader1004.wikimedia.org |
[production] |
| 08:04 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2052.codfw.wmnet'] |
[production] |
| 08:04 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2053.codfw.wmnet'] |
[production] |
| 08:04 |
<slyngshede@dns1004> |
END - running authdns-update |
[production] |
| 08:03 |
<godog> |
tools-nfs-3 is back - T404584 |
[tools] |
| 08:03 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1057 (re)pooling @ 60%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83913 and previous config saved to /var/cache/conftool/dbconfig/20251015-080327-root.json |
[production] |
| 08:03 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1052 (re)pooling @ 60%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83912 and previous config saved to /var/cache/conftool/dbconfig/20251015-080321-root.json |
[production] |
| 08:03 |
<slyngshede@dns1004> |
START - running authdns-update |
[production] |
| 08:02 |
<slyngs> |
Moving CAS/IDP/SSO to Trixie. |
[production] |
| 08:00 |
<godog> |
resize tools-nfs-3 - T404584 |
[tools] |
| 07:58 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2053.codfw.wmnet'] |
[production] |
| 07:57 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2054.codfw.wmnet'] |
[production] |
| 07:53 |
<mvernon@cumin1002> |
START - Cookbook sre.swift.roll-restart-reboot-swift-thanos-proxies rolling reboot on A:thanos-fe |
[production] |
| 07:50 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2054.codfw.wmnet'] |
[production] |
| 07:50 |
<elukey@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cp2055.codfw.wmnet'] |
[production] |
| 07:50 |
<elukey@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cp2055.codfw.wmnet'] |
[production] |
| 07:48 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1057 (re)pooling @ 50%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83910 and previous config saved to /var/cache/conftool/dbconfig/20251015-074821-root.json |
[production] |
| 07:48 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1052 (re)pooling @ 50%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83909 and previous config saved to /var/cache/conftool/dbconfig/20251015-074815-root.json |
[production] |
| 07:33 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1057 (re)pooling @ 30%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83907 and previous config saved to /var/cache/conftool/dbconfig/20251015-073316-root.json |
[production] |
| 07:33 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'es1052 (re)pooling @ 30%: Host provisioned T406488', diff saved to https://phabricator.wikimedia.org/P83906 and previous config saved to /var/cache/conftool/dbconfig/20251015-073309-root.json |
[production] |
| 07:28 |
<fceratto@cumin1002> |
START - Cookbook sre.mysql.pool es2032 gradually with 4 steps - Pool es2032.codfw.wmnet in after cloning |
[production] |