On 12/07/2021 17:16, Ulli Horlacher wrote:

On Mon 2021-07-12 (14:06), Graham Cobb wrote:

quoted

quoted

quoted

quoted

root@tsmsrvj:# snaprotate -v test 5 /data/fex/spool
$ btrfs subvolume snapshot -r /data/fex/spool /data/fex/spool/.snapshot/2021-07-10_0849.test
Create a readonly snapshot of '/data/fex/spool' in '/data/fex/spool/.snapshot/2021-07-10_0849.test'

I think this might be the source of the problem. Nested snapshots are
not a good idea, it causes various kinds of confusion.

I do not have nested snapshots anywhere.
/data/fex/spool is not a snapshot.

But it is the subvolume which is being snapshotted. What happens if you
put the snapshots somewhere that is not part of that subvolume? For
example, create /data/fex/snapshots, snapshot /data/fex/spool into a
snapshot in /data/fex/snapshots/spool/2021-07-10_0849.test, export
/data/fex/snapshots using NFS and mount /data/fex/snapshots on the client?

Same problem:

root@tsmsrvj:/etc# mount | grep data
/dev/sdb1 on /data type btrfs (rw,relatime,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/)

root@tsmsrvj:/etc# mkdir /data/snapshots /nfs/localhost/snapshots

root@tsmsrvj:/etc# btrfs subvolume snapshot -r /data/fex/spool /data/snapshots/fex_1
Create a readonly snapshot of '/data/fex/spool' in '/data/snapshots/fex_1'

root@tsmsrvj:/etc# btrfs subvolume snapshot -r /data/fex/spool /data/snapshots/fex_2
Create a readonly snapshot of '/data/fex/spool' in '/data/snapshots/fex_2'

root@tsmsrvj:/etc# btrfs subvolume list /data
ID 257 gen 1558 top level 5 path fex
ID 270 gen 1557 top level 257 path fex/spool
ID 272 gen 23 top level 270 path fex/spool/.snapshot/2021-03-07_1531.test
ID 273 gen 25 top level 270 path fex/spool/.snapshot/2021-03-07_1532.test
ID 274 gen 27 top level 270 path fex/spool/.snapshot/2021-03-07_1718.test
ID 394 gen 1470 top level 270 path fex/spool/.snapshot/2021-07-10_0849.test
ID 399 gen 1554 top level 270 path fex/spool/.snapshot/2021-07-12_1747.test
ID 400 gen 1556 top level 5 path snapshots/fex_1
ID 401 gen 1557 top level 5 path snapshots/fex_2

root@tsmsrvj:/etc# grep localhost /etc/exports 
/data/fex       localhost(rw,async,no_subtree_check,no_root_squash,crossmnt)
/data/snapshots localhost(rw,async,no_subtree_check,no_root_squash,crossmnt)

## ==> no nested subvolumes! different nfs exports

root@tsmsrvj:/etc# mount -o vers=3 localhost:/data/fex /nfs/localhost/fex
root@tsmsrvj:/etc# mount | grep localhost
localhost:/data/fex on /nfs/localhost/fex type nfs (rw,relatime,vers=3,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=127.0.0.1,mountvers=3,mountport=37961,mountproto=udp,local_lock=none,addr=127.0.0.1)

root@tsmsrvj:/etc# mount -o vers=3 localhost:/data/snapshots /nfs/localhost/snapshots
root@tsmsrvj:/etc# mount | grep localhost
localhost:/data/fex on /nfs/localhost/fex type nfs (rw,relatime,vers=3,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=127.0.0.1,mountvers=3,mountport=37961,mountproto=udp,local_lock=none,addr=127.0.0.1)
localhost:/data/fex on /nfs/localhost/snapshots type nfs (rw,relatime,vers=3,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=127.0.0.1,mountvers=3,mountport=37961,mountproto=udp,local_lock=none,addr=127.0.0.1)

## why localhost:/data/fex twice??

root@tsmsrvj:/etc# du -Hs /nfs/localhost/snapshots
du: WARNING: Circular directory structure.
This almost certainly means that you have a corrupted file system.
NOTIFY YOUR SYSTEM MANAGER.
The following directory is part of the cycle:
  /nfs/localhost/snapshots/spool

Sure. But it makes the useful operations work. du, find, ls -R, etc all
work properly on /nfs/localhost/fex.

When I go looking in the snapshots I am generally looking for which
version of a particular file I need to restore. For example, maybe I
want to find an old version of /nfs/localhost/fex/spool/some/file. I
would then find the best snapshot to use with:

ls -l /nfs/localhost/fex_snapshots/spool_*/some/file

which might show something like:

-rw-r--r-- 1 cobb me 2.8K 2018-04-03
/nfs/localhost/fex_snapshots/spool_20210703/some/file
-rw-r--r-- 1 cobb me 7 2021-07-06
/nfs/localhost/fex_snapshots/spool_20210706/some/file
-rw-r--r-- 1 cobb me 25 2021-07-12
/nfs/localhost/fex_snapshots/spool_20210712/some/file

So I could tell I need to restore the version from spool_20210703 if I
need the one with the old data in it, which got lost a few days ago.

This is exactly how I use NFS to access my btrbk snapshots stored on the
backup server. Of course, if you need to restore a whole subvolume you
are better of using btrfs send/receive to bring the snapshot back,
instead of using NFS - that preserves the btrfs features like reflinks.