|
2025-05-26
ยง
|
| 19:24 |
<aokoth@deploy1003> |
helmfile [aux-k8s-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 19:19 |
<denisse> |
Upgrading Grafana to v12.0.1 on grafana1002 - T395098 |
[production] |
| 19:19 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2211 (T395241)', diff saved to https://phabricator.wikimedia.org/P76447 and previous config saved to /var/cache/conftool/dbconfig/20250526-191912-fceratto.json |
[production] |
| 19:19 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2211.codfw.wmnet with reason: Maintenance |
[production] |
| 19:16 |
<denisse> |
Add Grafana v12.0.1 to reprepro for bookworm - T395098 |
[production] |
| 19:14 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2201.codfw.wmnet with reason: Maintenance |
[production] |
| 19:13 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2192 (T395241)', diff saved to https://phabricator.wikimedia.org/P76446 and previous config saved to /var/cache/conftool/dbconfig/20250526-191341-fceratto.json |
[production] |
| 18:58 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2192', diff saved to https://phabricator.wikimedia.org/P76445 and previous config saved to /var/cache/conftool/dbconfig/20250526-185832-fceratto.json |
[production] |
| 18:43 |
<aokoth@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on doc1003.eqiad.wmnet with reason: Bookworm |
[production] |
| 18:43 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2192', diff saved to https://phabricator.wikimedia.org/P76444 and previous config saved to /var/cache/conftool/dbconfig/20250526-184325-fceratto.json |
[production] |
| 18:28 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2192 (T395241)', diff saved to https://phabricator.wikimedia.org/P76443 and previous config saved to /var/cache/conftool/dbconfig/20250526-182817-fceratto.json |
[production] |
| 18:22 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2192 (T395241)', diff saved to https://phabricator.wikimedia.org/P76442 and previous config saved to /var/cache/conftool/dbconfig/20250526-182247-fceratto.json |
[production] |
| 18:22 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2192.codfw.wmnet with reason: Maintenance |
[production] |
| 18:22 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2178 (T395241)', diff saved to https://phabricator.wikimedia.org/P76441 and previous config saved to /var/cache/conftool/dbconfig/20250526-182221-fceratto.json |
[production] |
| 18:07 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2178', diff saved to https://phabricator.wikimedia.org/P76440 and previous config saved to /var/cache/conftool/dbconfig/20250526-180714-fceratto.json |
[production] |
| 17:52 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2178', diff saved to https://phabricator.wikimedia.org/P76439 and previous config saved to /var/cache/conftool/dbconfig/20250526-175207-fceratto.json |
[production] |
| 17:37 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2178 (T395241)', diff saved to https://phabricator.wikimedia.org/P76438 and previous config saved to /var/cache/conftool/dbconfig/20250526-173700-fceratto.json |
[production] |
| 17:29 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2178 (T395241)', diff saved to https://phabricator.wikimedia.org/P76437 and previous config saved to /var/cache/conftool/dbconfig/20250526-172912-fceratto.json |
[production] |
| 17:29 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2178.codfw.wmnet with reason: Maintenance |
[production] |
| 17:28 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171 (T395241)', diff saved to https://phabricator.wikimedia.org/P76436 and previous config saved to /var/cache/conftool/dbconfig/20250526-172844-fceratto.json |
[production] |
| 17:13 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171', diff saved to https://phabricator.wikimedia.org/P76435 and previous config saved to /var/cache/conftool/dbconfig/20250526-171338-fceratto.json |
[production] |
| 16:58 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171', diff saved to https://phabricator.wikimedia.org/P76434 and previous config saved to /var/cache/conftool/dbconfig/20250526-165831-fceratto.json |
[production] |
| 16:43 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171 (T395241)', diff saved to https://phabricator.wikimedia.org/P76433 and previous config saved to /var/cache/conftool/dbconfig/20250526-164324-fceratto.json |
[production] |
| 16:35 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2171 (T395241)', diff saved to https://phabricator.wikimedia.org/P76432 and previous config saved to /var/cache/conftool/dbconfig/20250526-163530-fceratto.json |
[production] |
| 16:35 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2171.codfw.wmnet with reason: Maintenance |
[production] |
| 16:35 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157 (T395241)', diff saved to https://phabricator.wikimedia.org/P76431 and previous config saved to /var/cache/conftool/dbconfig/20250526-163502-fceratto.json |
[production] |
| 16:19 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157', diff saved to https://phabricator.wikimedia.org/P76430 and previous config saved to /var/cache/conftool/dbconfig/20250526-161955-fceratto.json |
[production] |
| 16:15 |
<jgiannelos@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/changeprop: apply |
[production] |
| 16:14 |
<jgiannelos@deploy1003> |
helmfile [eqiad] START helmfile.d/services/changeprop: apply |
[production] |
| 16:14 |
<jgiannelos@deploy1003> |
helmfile [staging] DONE helmfile.d/services/changeprop: apply |
[production] |
| 16:13 |
<jgiannelos@deploy1003> |
helmfile [staging] START helmfile.d/services/changeprop: apply |
[production] |
| 16:07 |
<cgoubert@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/mw-cron: apply |
[production] |
| 16:07 |
<cgoubert@deploy1003> |
helmfile [eqiad] START helmfile.d/services/mw-cron: apply |
[production] |
| 16:04 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157', diff saved to https://phabricator.wikimedia.org/P76429 and previous config saved to /var/cache/conftool/dbconfig/20250526-160447-fceratto.json |
[production] |
| 15:49 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157 (T395241)', diff saved to https://phabricator.wikimedia.org/P76428 and previous config saved to /var/cache/conftool/dbconfig/20250526-154939-fceratto.json |
[production] |
| 15:49 |
<jgiannelos@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:48 |
<jgiannelos@deploy1003> |
helmfile [codfw] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:48 |
<jgiannelos@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:47 |
<jgiannelos@deploy1003> |
helmfile [eqiad] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:47 |
<jgiannelos@deploy1003> |
helmfile [staging] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:47 |
<jgiannelos@deploy1003> |
helmfile [staging] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:46 |
<jgiannelos@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:46 |
<jgiannelos@deploy1003> |
helmfile [eqiad] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:46 |
<volans@cumin1003> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts cirrussearch2111.codfw.wmnet |
[production] |
| 15:45 |
<volans@cumin1003> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host cirrussearch2111.codfw.wmnet |
[production] |
| 15:45 |
<jgiannelos@deploy1003> |
helmfile [staging] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:44 |
<jgiannelos@deploy1003> |
helmfile [staging] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:43 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2157 (T395241)', diff saved to https://phabricator.wikimedia.org/P76427 and previous config saved to /var/cache/conftool/dbconfig/20250526-154304-fceratto.json |
[production] |
| 15:42 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2157.codfw.wmnet with reason: Maintenance |
[production] |
| 15:32 |
<hnowlan@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply |
[production] |