Context

使用 Docker 拉取 MySQL 镜像时发现磁盘空间不够：no space left on device

root@ubuntu:~# docker pull mysql:5.7.29
5.7.29: Pulling from library/mysql
68ced04f60ab: Pull complete 
f9748e016a5c: Pull complete 
da54b038fed1: Pull complete 
6895ec5eb2c0: Pull complete 
111ba0647b87: Pull complete 
c1dce60f2f1a: Pull complete 
702ec598d0af: Pull complete 
63cca87a5d4d: Pull complete 
ec05b7b1c5c7: Extracting [==================================================>]  112.2MB/112.2MB
834b1d9f49b0: Download complete 
8ded6a30c87c: Download complete 
failed to register layer: Error processing tar file(exit status 1): write /usr/sbin/mysqld: no space left on device

Investigation

查看物理盘的占用情况

$ lsblk
NAME                      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
loop0                       7:0    0 55.5M  1 loop /snap/core18/2074
loop1                       7:1    0 55.4M  1 loop /snap/core18/2128
loop2                       7:2    0 61.9M  1 loop /snap/core20/1169
loop3                       7:3    0 70.3M  1 loop /snap/lxd/21029
loop4                       7:4    0 67.3M  1 loop /snap/lxd/21545
loop5                       7:5    0 61.8M  1 loop /snap/core20/1081
loop6                       7:6    0 32.3M  1 loop /snap/snapd/13170
loop8                       7:8    0 32.4M  1 loop /snap/snapd/13270
sda                         8:0    0  200G  0 disk
├─sda1                      8:1    0  512M  0 part /boot/efi
├─sda2                      8:2    0    1G  0 part /boot
└─sda3                      8:3    0 38.5G  0 part
  └─ubuntu--vg-ubuntu--lv 253:0    0   20G  0 lvm  /

查看文件系统磁盘空间的使用情况

root@ubuntu:~# df -h
Filesystem                         Size  Used Avail Use% Mounted on
udev                               451M     0  451M   0% /dev
tmpfs                               97M  1.3M   96M   2% /run
/dev/mapper/ubuntu--vg-ubuntu--lv  3.9G  3.4G  287M  93% /
tmpfs                              482M     0  482M   0% /dev/shm
tmpfs                              5.0M     0  5.0M   0% /run/lock
tmpfs                              482M     0  482M   0% /sys/fs/cgroup
/dev/loop0                          92M   92M     0 100% /snap/core/8689
/dev/loop1                          90M   90M     0 100% /snap/core/8268
/dev/sda2                          976M   77M  832M   9% /boot
tmpfs                               97M     0   97M   0% /run/user/0

查看LVM卷组的信息

root@ubuntu:~# vgdisplay
  --- Volume group ---
  VG Name               ubuntu-vg
  System ID             
  Format                lvm2
  Metadata Areas        1
  Metadata Sequence No  2
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                1
  Open LV               1
  Max PV                0
  Cur PV                1
  Act PV                1
  VG Size               <19.00 GiB
  PE Size               4.00 MiB
  Total PE              4863
  Alloc PE / Size       1024 / 4.00 GiB
  Free  PE / Size       3839 / <15.00 GiB
  VG UUID               hZPoxm-kSBU-fmKs-2yXD-hHoe-3T7e-PCIFSe

Free PE / Size 3839 / <15.00 GiB，这是还可以扩充的大小

Solution

扩容 logical volumn

使用命令进行对 logical volumn 扩容

lvextend -L 10G /dev/mapper/ubuntu--vg-ubuntu--lv      //增大或减小至19G
lvextend -L +10G /dev/mapper/ubuntu--vg-ubuntu--lv     //增加10G
lvreduce -L -10G /dev/mapper/ubuntu--vg-ubuntu--lv     //减小10G
lvresize -l  +100%FREE /dev/mapper/ubuntu--vg-ubuntu--lv   //按百分比扩容

#//执行调整
$ sudo resize2fs /dev/mapper/ubuntu--vg-ubuntu--lv
resize2fs 1.45.5 (07-Jan-2020)
Filesystem at /dev/mapper/ubuntu--vg-ubuntu--lv is mounted on /; on-line resizing required
old_desc_blocks = 4, new_desc_blocks = 5
The filesystem on /dev/mapper/ubuntu--vg-ubuntu--lv is now 8978432 (4k) blocks long.

具体操作如下

# 按百分比对 volumn 进行扩容
root@ubuntu:~# sudo lvextend -l +100%FREE /dev/mapper/ubuntu--vg-ubuntu--lv
  Size of logical volume ubuntu-vg/ubuntu-lv changed from 4.00 GiB (1024 extents) to <19.00 GiB (4863 extents).
  Logical volume ubuntu-vg/ubuntu-lv successfully resized.

# 刷新分区 
root@ubuntu:~# sudo resize2fs /dev/mapper/ubuntu--vg-ubuntu--lv 
resize2fs 1.44.1 (24-Mar-2018)
Filesystem at /dev/mapper/ubuntu--vg-ubuntu--lv is mounted on /; on-line resizing required
old_desc_blocks = 3, new_desc_blocks = 3
The filesystem on /dev/mapper/ubuntu--vg-ubuntu--lv is now 4979712 (4k) blocks long.

# 查看文件系统磁盘空间的使用情况（发现此时已经扩容到19G）
root@ubuntu:~# df -h
Filesystem                         Size  Used Avail Use% Mounted on
udev                               451M     0  451M   0% /dev
tmpfs                               97M  1.3M   96M   2% /run
/dev/mapper/ubuntu--vg-ubuntu--lv   19G  3.9G   14G  22% /
tmpfs                              482M     0  482M   0% /dev/shm
tmpfs                              5.0M     0  5.0M   0% /run/lock
tmpfs                              482M     0  482M   0% /sys/fs/cgroup
/dev/loop0                          92M   92M     0 100% /snap/core/8689
/dev/loop1                          90M   90M     0 100% /snap/core/8268
/dev/sda2                          976M   77M  832M   9% /boot
tmpfs                               97M     0   97M   0% /run/user/0

再次查看LVM卷组的信息（可扩容的空间已经成为0）

$ sudo vgdisplay
  --- Volume group ---
  VG Name               ubuntu-vg
  System ID             
  Format                lvm2
  Metadata Areas        1
  Metadata Sequence No  4
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                1
  Open LV               1
  Max PV                0
  Cur PV                1
  Act PV                1
  VG Size               <19.00 GiB
  PE Size               4.00 MiB
  Total PE              4863
  Alloc PE / Size       4863 / <19.00 GiB
  Free  PE / Size       0 / 0   
  VG UUID               hZPoxm-kSBU-fmKs-2yXD-hHoe-3T7e-PCIFSe

扩容 partition

如果 Free PE已经是0了，但是磁盘中仍然有可用的空间，比如

$ lsblk
NAME                      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
loop0                       7:0    0 55.4M  1 loop /snap/core18/1944
loop1                       7:1    0 55.4M  1 loop /snap/core18/2128
loop2                       7:2    0 61.8M  1 loop /snap/core20/1081
loop3                       7:3    0 67.3M  1 loop /snap/lxd/21545
loop4                       7:4    0 70.3M  1 loop /snap/lxd/21029
loop5                       7:5    0 61.9M  1 loop /snap/core20/1169
loop6                       7:6    0 32.4M  1 loop /snap/snapd/13270
loop7                       7:7    0 32.3M  1 loop /snap/snapd/13170
sda                         8:0    0  200G  0 disk
|-sda1                      8:1    0  512M  0 part /boot/efi
|-sda2                      8:2    0    1G  0 part /boot
`-sda3                      8:3    0 48.5G  0 part
  `-ubuntu--vg-ubuntu--lv 253:0    0 48.5G  0 lvm  /

可以看到 sda有200G，其中sda3只用了48.5G。

注意，不能像下面这样直接resize，因为当前partition正在被使用：

$ sudo resize2fs /dev/sda3 50G
[sudo] password for sw:
resize2fs 1.45.5 (07-Jan-2020)
resize2fs: Device or resource busy while trying to open /dev/sda3
Couldn't find valid filesystem superblock.

Partitions can’t be modified while they’re in use – partitions in use have a key icon next to them. If a partition is mounted, unmount it by clicking the eject button in the file manager.

因此，我们需要使用一个安装了Ubuntu Desktop 的 U盘，来resize partition

https://www.howtogeek.com/114503/how-to-resize-your-ubuntu-partitions/

使用GParted resize 成 97.7GB后，重启Ubuntu并直接进入系统（而不是进入U盘）：

$ lsblk
NAME                      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
loop0                       7:0    0 55.5M  1 loop /snap/core18/2074
loop1                       7:1    0 55.4M  1 loop /snap/core18/2128
loop2                       7:2    0 61.8M  1 loop /snap/core20/1081
loop3                       7:3    0 61.9M  1 loop /snap/core20/1169
loop4                       7:4    0 32.3M  1 loop /snap/snapd/13170
loop5                       7:5    0 32.4M  1 loop /snap/snapd/13270
loop6                       7:6    0 70.3M  1 loop /snap/lxd/21029
loop7                       7:7    0 67.3M  1 loop /snap/lxd/21545
sda                         8:0    0  200G  0 disk
|-sda1                      8:1    0  512M  0 part /boot/efi
|-sda2                      8:2    0    1G  0 part /boot
`-sda3                      8:3    0 97.7G  0 part
  `-ubuntu--vg-ubuntu--lv 253:0    0 38.5G  0 lvm  /

如果是操作一个remote的物理机，即无法插入U盘，参考

https://askubuntu.com/questions/109215/resize-disk-partition-on-remote-vm-server/109224#109224

# 按百分比对 volumn 进行扩容
$ sudo lvextend -l +100%FREE /dev/mapper/ubuntu--vg-ubuntu--lv
  Size of logical volume ubuntu-vg/ubuntu-lv changed from <38.50 GiB (9855 extents) to <97.66 GiB (25000 extents).
  Logical volume ubuntu-vg/ubuntu-lv successfully resized.

# 刷新分区 
$ sudo resize2fs /dev/mapper/ubuntu--vg-ubuntu--lv 
resize2fs 1.45.5 (07-Jan-2020)
Filesystem at /dev/mapper/ubuntu--vg-ubuntu--lv is mounted on /; on-line resizing required
old_desc_blocks = 3, new_desc_blocks = 13
The filesystem on /dev/mapper/ubuntu--vg-ubuntu--lv is now 25600000 (4k) blocks long.

# 查看文件系统磁盘空间的使用情况（发现此时已经扩容到19G）
$ df -h
Filesystem                         Size  Used Avail Use% Mounted on
udev                               1.9G     0  1.9G   0% /dev
tmpfs                              394M  992K  393M   1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv   96G   11G   82G  12% /
tmpfs                              2.0G     0  2.0G   0% /dev/shm
tmpfs                              5.0M     0  5.0M   0% /run/lock
tmpfs                              2.0G     0  2.0G   0% /sys/fs/cgroup
/dev/sda2                          976M  300M  609M  34% /boot
/dev/sda1                          511M  5.3M  506M   2% /boot/efi
/dev/loop1                          56M   56M     0 100% /snap/core18/2128
/dev/loop2                          62M   62M     0 100% /snap/core20/1081
/dev/loop0                          56M   56M     0 100% /snap/core18/2074
/dev/loop3                          62M   62M     0 100% /snap/core20/1169
/dev/loop4                          33M   33M     0 100% /snap/snapd/13170
/dev/loop5                          33M   33M     0 100% /snap/snapd/13270
/dev/loop6                          71M   71M     0 100% /snap/lxd/21029
/dev/loop7                          68M   68M     0 100% /snap/lxd/21545
tmpfs                              394M     0  394M   0% /run/user/1000

找到并删除大文件

找到大文件

$ sudo -i
# -a display all (including files and directories)
# -h print human readable format
$ sudo du -ah / 2>/dev/null | sort -n -r | head -n 20

$ find / -type f -size +100M  -print0 | xargs -0 du -h | sort -nr

# 查找耗用最大磁盘空间的顶级10文件和目录
# -c produce a grand total
# -s display only a total for each argument
# -k like --block-size=1K
$ du -cks / | sort -rn | head

$ du -ahx / | sort -rh | head -5

查看占用空间大的文件夹

$ du -h / 2>/dev/null | sort -hr | head -n 20

$ du -ha --max-depth=100 / 2>/dev/null | sort -hr | head -n 20

【Linux】磁盘空间不足

Context

Investigation

Solution

扩容 logical volumn

扩容 partition

找到并删除大文件

Reference

FEATURED TAGS

FRIENDS

TOC