| 
      
        2024-11-15
      
      ยง
     | 
  
    
  | 17:17 | 
  <aborrero@cloudcumin1001> | 
  START - Cookbook wmcs.openstack.tofu running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:14 | 
  <aborrero@cloudcumin1001> | 
  END (FAIL) - Cookbook wmcs.openstack.tofu (exit_code=99) running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:13 | 
  <aborrero@cloudcumin1001> | 
  START - Cookbook wmcs.openstack.tofu running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:10 | 
  <aborrero@cloudcumin1001> | 
  END (FAIL) - Cookbook wmcs.openstack.tofu (exit_code=99) running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:09 | 
  <aborrero@cloudcumin1001> | 
  START - Cookbook wmcs.openstack.tofu running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:06 | 
  <aborrero@cloudcumin1001> | 
  END (FAIL) - Cookbook wmcs.openstack.tofu (exit_code=99) running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:06 | 
  <aborrero@cloudcumin1001> | 
  START - Cookbook wmcs.openstack.tofu running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:06 | 
  <aborrero@cloudcumin1001> | 
  END (FAIL) - Cookbook wmcs.openstack.tofu (exit_code=99) running tofu plan+apply for main branch | 
  [admin] | 
            
  | 17:05 | 
  <aborrero@cloudcumin1001> | 
  START - Cookbook wmcs.openstack.tofu running tofu plan+apply for main branch | 
  [admin] | 
            
  | 16:58 | 
  <mfossati@deploy2002> | 
  Finished deploy [airflow-dags/platform_eng@82083c4]: image suggestions hotfix - section titles denylist dependency (duration: 01m 58s) | 
  [production] | 
            
  | 16:57 | 
  <taavi> | 
  copy python3-flask-{keystone,oslolog} from bullseye-wikimedia to bookworm-wikimedia | 
  [production] | 
            
  | 16:56 | 
  <mfossati@deploy2002> | 
  Started deploy [airflow-dags/platform_eng@82083c4]: image suggestions hotfix - section titles denylist dependency | 
  [production] | 
            
  | 16:30 | 
  <aborrero@cloudcumin2001> | 
  END (PASS) - Cookbook wmcs.openstack.restart_openstack (exit_code=0) | 
  [admin] | 
            
  | 16:29 | 
  <arturo> | 
  [codfw1dev] restart rabbitmq and designate | 
  [admin] | 
            
  | 16:29 | 
  <aborrero@cloudcumin2001> | 
  START - Cookbook wmcs.openstack.restart_openstack | 
  [admin] | 
            
  | 16:27 | 
  <herron@cumin2002> | 
  conftool action : set/pooled=yes; selector: name=aux-k8s-worker1005.eqiad.wmnet,cluster=aux-k8s,service=kubesvc | 
  [production] | 
            
  | 16:27 | 
  <herron@cumin2002> | 
  conftool action : set/weight=10; selector: name=aux-k8s-worker1005.eqiad.wmnet,cluster=aux-k8s,service=kubesvc | 
  [production] | 
            
  | 16:22 | 
  <herron@cumin2002> | 
  conftool action : set/pooled=yes; selector: name=aux-k8s-worker1004.eqiad.wmnet,cluster=aux-k8s,service=kubesvc | 
  [production] | 
            
  | 16:22 | 
  <herron@cumin2002> | 
  conftool action : set/weight=10; selector: name=aux-k8s-worker1004.eqiad.wmnet,cluster=aux-k8s,service=kubesvc | 
  [production] | 
            
  | 16:09 | 
  <sukhe@puppetserver1001> | 
  conftool action : set/pooled=yes; selector: name=cp4043.ulsfo.wmnet [reason: ATS fixed] | 
  [production] | 
            
  | 16:08 | 
  <sukhe@cumin1002> | 
  END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for cp4043.ulsfo.wmnet | 
  [production] | 
            
  | 16:08 | 
  <sukhe@cumin1002> | 
  START - Cookbook sre.hosts.remove-downtime for cp4043.ulsfo.wmnet | 
  [production] | 
            
  | 16:06 | 
  <sukhe@cumin1002> | 
  END (PASS) - Cookbook sre.cdn.roll-upgrade-ats (exit_code=0) Rolling upgrade/restart of Apache Traffic Server on P{cp4051*} and A:cp for 9.2.6-1wm2 | 
  [production] | 
            
  | 16:03 | 
  <sukhe@cumin1002> | 
  START - Cookbook sre.cdn.roll-upgrade-ats Rolling upgrade/restart of Apache Traffic Server on P{cp4051*} and A:cp for 9.2.6-1wm2 | 
  [production] | 
            
  | 16:00 | 
  <sukhe> | 
  reprepro -C main include bullseye-wikimedia trafficserver_9.2.6-1wm2_amd64.changes: T379797 | 
  [production] | 
            
  | 15:47 | 
  <arnaudb@cumin1002> | 
  END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4 days, 0:00:00 on db2230.codfw.wmnet,db1125.eqiad.wmnet with reason: testing stuff on test-s4 | 
  [production] | 
            
  | 15:47 | 
  <arnaudb@cumin1002> | 
  START - Cookbook sre.hosts.downtime for 4 days, 0:00:00 on db2230.codfw.wmnet,db1125.eqiad.wmnet with reason: testing stuff on test-s4 | 
  [production] | 
            
  | 15:42 | 
  <arnaudb@cumin1002> | 
  END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw | 
  [production] | 
            
  | 15:41 | 
  <arnaudb@cumin1002> | 
  START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw | 
  [production] | 
            
  | 15:40 | 
  <arnaudb@cumin1002> | 
  END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad | 
  [production] | 
            
  | 15:39 | 
  <arnaudb@cumin1002> | 
  START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad | 
  [production] | 
            
  | 15:39 | 
  <arnaudb@cumin1002> | 
  END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad | 
  [production] | 
            
  | 15:38 | 
  <brouberol@deploy2002> | 
  helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-platform-eng: apply | 
  [production] | 
            
  | 15:38 | 
  <arnaudb@cumin1002> | 
  START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad | 
  [production] | 
            
  | 15:37 | 
  <brouberol@deploy2002> | 
  helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-platform-eng: apply | 
  [production] | 
            
  | 15:35 | 
  <brouberol@deploy2002> | 
  helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'. | 
  [production] | 
            
  | 15:34 | 
  <brouberol@deploy2002> | 
  helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'. | 
  [production] | 
            
  | 14:05 | 
  <fnegri@cloudcumin1001> | 
  END (PASS) - Cookbook wmcs.vps.refresh_puppet_certs (exit_code=0) on tools-db-5.tools.eqiad1.wikimedia.cloud (T352206) | 
  [tools] | 
            
  | 14:03 | 
  <fnegri@cloudcumin1001> | 
  START - Cookbook wmcs.vps.refresh_puppet_certs on tools-db-5.tools.eqiad1.wikimedia.cloud (T352206) | 
  [tools] | 
            
  | 14:03 | 
  <fnegri@cloudcumin1001> | 
  END (PASS) - Cookbook wmcs.vps.create_instance_with_prefix (exit_code=0) with prefix 'tools-db' (T352206) | 
  [tools] | 
            
  | 13:59 | 
  <ayounsi@cumin1002> | 
  END (PASS) - Cookbook sre.dns.netbox (exit_code=0) | 
  [production] | 
            
  | 13:59 | 
  <ayounsi@cumin1002> | 
  END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Remove e8 lo0 IP - ayounsi@cumin1002" | 
  [production] | 
            
  | 13:59 | 
  <ayounsi@cumin1002> | 
  START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Remove e8 lo0 IP - ayounsi@cumin1002" | 
  [production] | 
            
  | 13:57 | 
  <fnegri@cloudcumin1001> | 
  START - Cookbook wmcs.vps.create_instance_with_prefix with prefix 'tools-db' (T352206) | 
  [tools] | 
            
  | 13:57 | 
  <fnegri@cloudcumin1001> | 
  END (PASS) - Cookbook wmcs.openstack.quota_increase (exit_code=0) (T352206) | 
  [tools] | 
            
  | 13:57 | 
  <fnegri@cloudcumin1001> | 
  START - Cookbook wmcs.openstack.quota_increase (T352206) | 
  [tools] | 
            
  | 13:55 | 
  <ayounsi@cumin1002> | 
  START - Cookbook sre.dns.netbox | 
  [production] | 
            
  | 13:55 | 
  <ayounsi@cumin1002> | 
  END (FAIL) - Cookbook sre.dns.netbox (exit_code=99) | 
  [production] | 
            
  | 13:52 | 
  <ayounsi@cumin1002> | 
  START - Cookbook sre.dns.netbox | 
  [production] | 
            
  | 13:50 | 
  <fnegri@cloudcumin1001> | 
  END (FAIL) - Cookbook wmcs.vps.create_instance_with_prefix (exit_code=99) with prefix 'tools-db' (T352206) | 
  [tools] |