[TOC]
背景
前段时间平台上一台虚拟机（10.xxx.xxx.224）因前期宿主机未配置LVM过滤器，宿主机同步接管了虚拟机LVM信息，在进行整改清理阶段导致虚拟机内LVM丢失，逻辑卷无法正常挂载。该系统使用LVM存储数据量较大，足有3、40T数据大小，如果这些数据真丢失无法找回，影响后果会非常严重，记录下这次虚拟机LVM信息修复还原过程。
– 平台：OpenStack(Pike)
– 存储：
– ceph 12.2.10（Luminous）
– 华为5310v5 SAN
- 虚拟机信息
[root@hostname]# nova list --all --ip 10.xxx.xxx.224
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+-----------------------+
| ID | Name | Tenant ID | Status | Task State | Power State | Networks |
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+-----------------------+
| d6b961ae-3df2-46d1-8128-bfdca907f0bd | JXGLPT-3 | 4489501ffc5c45d0babe4274360d2151 | ACTIVE | - | Running | vlan461=10.xxx.xxx.224 |
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+-----------------------+
- 虚拟机所在宿主机信息及实例名称
[root@hostname]# nova show d6b961ae-3df2-46d1-8128-bfdca907f0bd
...
| OS-EXT-SRV-ATTR:host | COM017 |
| OS-EXT-SRV-ATTR:hostname | jxglpt-3 |
| OS-EXT-SRV-ATTR:hypervisor_hostname | COM017 |
| OS-EXT-SRV-ATTR:instance_name | instance-00000dda |
| id | d6b961ae-3df2-46d1-8128-bfdca907f0bd |
- 挂载卷信息
[root@hostname]# nova volume-attachments d6b961ae-3df2-46d1-8128-bfdca907f0bd
+--------------------------------------+----------+--------------------------------------+--------------------------------------+
| ID | DEVICE | SERVER ID | VOLUME ID |
+--------------------------------------+----------+--------------------------------------+--------------------------------------+
| b07ab1b8-4f96-4555-badc-a2c520e4df32 | /dev/vda | d6b961ae-3df2-46d1-8128-bfdca907f0bd | b07ab1b8-4f96-4555-badc-a2c520e4df32 |
| 9ee56873-0b56-4755-bf76-f588bbab79ff | /dev/vdb | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 9ee56873-0b56-4755-bf76-f588bbab79ff |
| f166d752-8f8b-40f6-be5d-0df857c78c25 | /dev/vdc | d6b961ae-3df2-46d1-8128-bfdca907f0bd | f166d752-8f8b-40f6-be5d-0df857c78c25 |
| ba8e07db-2499-4ece-8c9d-18cce929bac3 | /dev/vdd | d6b961ae-3df2-46d1-8128-bfdca907f0bd | ba8e07db-2499-4ece-8c9d-18cce929bac3 |
| 35eab613-cfd8-49c3-bec0-d627221f2185 | /dev/vde | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 35eab613-cfd8-49c3-bec0-d627221f2185 |
| e23a386b-a1c5-4c97-aac0-a75976bfbfdd | /dev/vdf | d6b961ae-3df2-46d1-8128-bfdca907f0bd | e23a386b-a1c5-4c97-aac0-a75976bfbfdd |
| 3dc03dfb-a6d5-439b-b78b-e6259cd3e1bf | /dev/vdg | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 3dc03dfb-a6d5-439b-b78b-e6259cd3e1bf |
| 0427c3f1-23dd-479b-b5b2-30043754937b | /dev/vdh | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 0427c3f1-23dd-479b-b5b2-30043754937b |
| e129f4e0-89c3-4f51-a1c9-32767b90f4a7 | /dev/vdi | d6b961ae-3df2-46d1-8128-bfdca907f0bd | e129f4e0-89c3-4f51-a1c9-32767b90f4a7 |
| aee0b85a-3be8-4e29-b938-53f6528b8466 | /dev/vdj | d6b961ae-3df2-46d1-8128-bfdca907f0bd | aee0b85a-3be8-4e29-b938-53f6528b8466 |
| 0bc61058-778a-405e-8bb9-48366950852b | /dev/vdk | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 0bc61058-778a-405e-8bb9-48366950852b |
| 796339b4-2047-408e-93c4-2a6a9ec74021 | /dev/vdl | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 796339b4-2047-408e-93c4-2a6a9ec74021 |
| be51bef6-5612-4e43-b076-f8bf4d19890f | /dev/vdm | d6b961ae-3df2-46d1-8128-bfdca907f0bd | be51bef6-5612-4e43-b076-f8bf4d19890f |
| edc0b346-b85f-4adb-814c-a457308e7731 | /dev/vdn | d6b961ae-3df2-46d1-8128-bfdca907f0bd | edc0b346-b85f-4adb-814c-a457308e7731 |
| a827106c-7d9c-4b14-88f3-98cc6b3a0a90 | /dev/vdo | d6b961ae-3df2-46d1-8128-bfdca907f0bd | a827106c-7d9c-4b14-88f3-98cc6b3a0a90 |
| 2aa41500-95e3-4cac-b770-d0365a40a3e1 | /dev/vdp | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 2aa41500-95e3-4cac-b770-d0365a40a3e1 |
| 3170184f-dc54-4f94-8895-e42de994f7d1 | /dev/vdq | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 3170184f-dc54-4f94-8895-e42de994f7d1 |
| 75c96474-3084-4fd1-a58d-be3fc75d3d73 | /dev/vdr | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 75c96474-3084-4fd1-a58d-be3fc75d3d73 |
| 6d276374-29df-415d-9765-e4499ebfde10 | /dev/vds | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 6d276374-29df-415d-9765-e4499ebfde10 |
| 984d05de-43bf-450d-85fd-d29b96fffd8e | /dev/vdt | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 984d05de-43bf-450d-85fd-d29b96fffd8e |
| 8ea55edd-f3e2-4633-9d1a-37cf9bd5f34e | /dev/vdu | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 8ea55edd-f3e2-4633-9d1a-37cf9bd5f34e |
| 8f536e21-bdab-4995-b822-b1780c6c66aa | /dev/vdv | d6b961ae-3df2-46d1-8128-bfdca907f0bd | 8f536e21-bdab-4995-b822-b1780c6c66aa |
+--------------------------------------+----------+--------------------------------------+--------------------------------------+
- 虚拟机实例磁盘挂载信息
[root@COM017 ~]# virsh domblklist instance-00000dda
Target Source
------------------------------------------------
hda vms/d6b961ae-3df2-46d1-8128-bfdca907f0bd_disk.config
hdb -
vda volumes/volume-b07ab1b8-4f96-4555-badc-a2c520e4df32
vdb volumes/volume-9ee56873-0b56-4755-bf76-f588bbab79ff
vdc /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8713700000002
vdd /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8718d00000003
vde /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8722d00000004
vdf /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d872ae00000005
vdg /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8731f00000006
vdh /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8739100000007
vdi /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8740300000008
vdj /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8748d00000009
vdk /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d874e60000000a
vdl /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8755a0000000b
vdm /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d875db0000000c
vdn /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8764c0000000d
vdo /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d876ad0000000e
vdp /dev/disk/by-id/dm-uuid-mpath-36bc76c5100b2de5711d8773e0000000f
vdq /dev/disk/by-id/dm-uuid-mpath-3600507670881094d3000000000000034
vdr /dev/disk/by-id/dm-uuid-mpath-3600507670881094d3000000000000035
vds /dev/disk/by-id/dm-uuid-mpath-3600507670881094d3000000000000036
vdt /dev/disk/by-id/dm-uuid-mpath-3600507670881094d3000000000000039
vdu /dev/disk/by-id/dm-uuid-mpath-3600507670881094d3000000000000038
vdv /dev/disk/by-id/dm-uuid-mpath-3600507670881094d3000000000000037
修复操作
备份COM017上LVM信息
平时记得经常备份是好习惯，指不定哪天就是救命稻草
cp -a /etc/lvm/archive /etc/lvm/archive-bak`date +%Y%m%d%H%M`
cp -a /etc/lvm/backup /etc/lvm/backup-bak`date +%Y%m%d%H%M`
登录宿主机COM017并拷贝LVM信息至虚拟机（10.xxx.xxx.224）
scp -P22 -r /root/instance-00000dda [email protected]:/root/
登录虚拟机（10.xxx.xxx.224）并备份当前LVM信息
cp -a /etc/lvm/archive /etc/lvm/archive-bak`date +%Y%m%d%H%M`
cp -a /etc/lvm/backup /etc/lvm/backup-bak`date +%Y%m%d%H%M`
删除虚拟机（10.xxx.xxx.224）原LVM信息
rm /etc/lvm/archive/data_vg_00000-1725208148.vg
rm /etc/lvm/archive/data_vg_00001-1111777997.vg
rm /etc/lvm/backup/data_vg
还原宿主机接管该虚拟机LVM信息
cp /root/instance-00000dda/lvm/archive/data_vg_00000-265816890.vg /etc/lvm/archive/
cp /root/instance-00000dda/lvm/backup/data_vg /etc/lvm/backup/
事后继续排查证明虚拟机上原LVM配置并未丢失，可以直接用原虚拟机上
/etc/lvm/backup/进行恢复还原
扫描PV信息
pvscan
从备份文件还原创建PV
确认主机下磁盘UUID一致
pvcreate -ff --uuid GS2RUu-QPJb-uegb-G3dg-D6GL-ZJ15-2MjLJI --restorefile /etc/lvm/backup/data_vg /dev/vdc
pvcreate -ff --uuid Jnaol0-L25i-0yQv-QIgb-wtVL-Wr4G-mUWccV --restorefile /etc/lvm/backup/data_vg /dev/vdd
pvcreate -ff --uuid PH86lF-V2dc-3m9y-aE0E-ZjEo-3lW5-qcVw0Z --restorefile /etc/lvm/backup/data_vg /dev/vde
pvcreate -ff --uuid u12Pbo-25ci-S81R-Cmlv-jrjx-HjcB-5Vpm1c --restorefile /etc/lvm/backup/data_vg /dev/vdf
pvcreate -ff --uuid bN3dTo-qRHs-QrFG-CYtw-lCiN-0Yum-7nyYnK --restorefile /etc/lvm/backup/data_vg /dev/vdg
pvcreate -ff --uuid z9Ghvv-MgGY-rchx-ccTz-Rtxq-UQBP-mOnTPe --restorefile /etc/lvm/backup/data_vg /dev/vdh
pvcreate -ff --uuid 08HBDH-S0Yx-hTP1-h59g-QObS-BDzK-bjTzuI --restorefile /etc/lvm/backup/data_vg /dev/vdi
pvcreate -ff --uuid CS8GPs-IbDA-zWpL-TBVD-tyqe-f17t-p2DctO --restorefile /etc/lvm/backup/data_vg /dev/vdj
pvcreate -ff --uuid gz1pmW-HHfX-o6df-B7fG-Qfbo-GnEN-lcFIXP --restorefile /etc/lvm/backup/data_vg /dev/vdk
pvcreate -ff --uuid idgW1b-kUsn-yfc7-0olp-5Ejn-gZYZ-qEkp2x --restorefile /etc/lvm/backup/data_vg /dev/vdl
pvcreate -ff --uuid jnebVC-OX1x-aOXq-BJat-tEpU-mdO1-GtewPr --restorefile /etc/lvm/backup/data_vg /dev/vdm
pvcreate -ff --uuid iwRjfv-f7K9-JcxO-j9Wz-nCd3-iPSB-ddryL8 --restorefile /etc/lvm/backup/data_vg /dev/vdn
pvcreate -ff --uuid Y0ofdi-gdRW-q8Ff-sy26-1aiz-k6xg-K9VSUC --restorefile /etc/lvm/backup/data_vg /dev/vdo
pvcreate -ff --uuid o8ezma-MgiS-e6Ug-R8X5-YBWa-er8L-4uZnDt --restorefile /etc/lvm/backup/data_vg /dev/vdp
pvcreate -ff --uuid RnvaGK-IAG4-zjYQ-TPdJ-QsVP-gjkg-nwSYZm --restorefile /etc/lvm/backup/data_vg /dev/vdq
pvcreate -ff --uuid copWTz-9DLK-Piao-IB8u-m8Y0-KUPp-637Wpa --restorefile /etc/lvm/backup/data_vg /dev/vdr
pvcreate -ff --uuid 7CWWZk-lj8v-GpNU-KINs-lg3o-xyBe-XYSWfO --restorefile /etc/lvm/backup/data_vg /dev/vds
pvcreate -ff --uuid 7c1hAN-AVzj-bvGn-HaC8-KlCj-aUUT-Cd780w --restorefile /etc/lvm/backup/data_vg /dev/vdt
pvcreate -ff --uuid URO7fB-XQgF-16pf-I4f3-O1XC-1471-sW9e4E --restorefile /etc/lvm/backup/data_vg /dev/vdu
pvcreate -ff --uuid dE3wRo-DAM0-MJCK-a5Ag-p5rn-jbZ9-p3BvuO --restorefile /etc/lvm/backup/data_vg /dev/vdv
查看PV信息
pvscan
扫描VG信息
vgscan
恢复卷组
vgcfgrestore -f /etc/lvm/backup/data_vg data_vg --test # 试运行
vgcfgrestore -f /etc/lvm/backup/data_vg data_vg
查看VG信息
vgscan
激活卷组(假如卷组未激活)
vgchange -ay data_vg
扫描LV信息
lvscan
激活LV（假如逻辑卷未激活）
lvchange -ay data_vg/data_lv
验证
此时虚拟机LVM信息已恢复完成，挂载LV卷验证运行服务及数据完整情况。
总结
整个处理过程就是：
1. 检查/etc/lvm/backup/下VG卷组信息是否完整正确；
2. 通过/etc/lvm/backup/下VG卷组信息还原创建PV物理卷；
3. 恢复VG卷组；
4. 激活VG卷组、LV逻辑卷。
早年在IBM网站看过一篇LVM灾难修复的文章，现在链接已失效，网上有人转载过写的比较好，这次在生产环境遇到故障脑海里想到当年看的这篇文章，最后成功恢复找回数据。
以上就是这次LVM灾难处理修复的一个案例，这次幸运的是LVM卷组信息完整并未丢失得以数据恢复成功。日常运维还是得记住养成备份习惯，在使用虚拟机时宿主机记得配置LVM过滤器。