From dee03c8d5c0b86cf51865090bec203419a3008a9 Mon Sep 17 00:00:00 2001 From: Kotresh HR Date: Mon, 15 Feb 2021 21:56:51 +0530 Subject: [PATCH] pybind/ceph_volume_client: Update the 'volumes' key to 'subvolumes' in auth metadata file The older auth metadata files before nautilus release stores the authorized subvolumes using the 'volumes' key. As the notion of 'subvolumes' brought in by mgr/volumes, it makes sense to use 'subvolumes' key. This patch would be tranparently update 'volumes' key to 'subvolumes' and newer auth metadata files would store them with 'subvolumes' key. Also fails the deauthorize if the auth-id doesn't exist. Fixes: https://tracker.ceph.com/issues/49294 Signed-off-by: Kotresh HR --- qa/tasks/cephfs/test_volume_client.py | 435 ++++++++++++++++++++++++++ src/pybind/ceph_volume_client.py | 41 ++- 2 files changed, 461 insertions(+), 15 deletions(-) diff --git a/qa/tasks/cephfs/test_volume_client.py b/qa/tasks/cephfs/test_volume_client.py index f889421d9fc8c..37abb9dde7ad8 100644 --- a/qa/tasks/cephfs/test_volume_client.py +++ b/qa/tasks/cephfs/test_volume_client.py @@ -1089,6 +1089,441 @@ vc.disconnect() auth_id=guestclient["auth_id"], ))) + def test_update_old_style_auth_metadata_to_new_during_recover(self): + """ + From nautilus onwards 'volumes' created by ceph_volume_client were + renamed and used as CephFS subvolumes accessed via the ceph-mgr + interface. Hence it makes sense to store the subvolume data in + auth-metadata file with 'subvolumes' key instead of 'volumes' key. + This test validates the transparent update of 'volumes' key to + 'subvolumes' key in auth metadata file during recover. + """ + volumeclient_mount = self.mounts[1] + volumeclient_mount.umount_wait() + + # Configure volumeclient_mount as the handle for driving volumeclient. + self._configure_vc_auth(volumeclient_mount, "manila") + + group_id = "groupid" + volume_id = "volumeid" + + guestclient = { + "auth_id": "guest", + "tenant_id": "tenant", + } + + # Create a volume. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.create_volume(vp, 1024*1024*10) + """.format( + group_id=group_id, + volume_id=volume_id, + ))) + + # Check that volume metadata file is created on volume creation. + vol_metadata_filename = "_{0}:{1}.meta".format(group_id, volume_id) + self.assertIn(vol_metadata_filename, self.mounts[0].ls("volumes")) + + # Authorize 'guestclient' access to the volume. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.authorize(vp, "{auth_id}", tenant_id="{tenant_id}") + """.format( + group_id=group_id, + volume_id=volume_id, + auth_id=guestclient["auth_id"], + tenant_id=guestclient["tenant_id"] + ))) + + # Check that auth metadata file for auth ID 'guest' is created. + auth_metadata_filename = "${0}.meta".format(guestclient["auth_id"]) + self.assertIn(auth_metadata_filename, self.mounts[0].ls("volumes")) + + # Replace 'subvolumes' to 'volumes', old style auth-metadata file + self.mounts[0].run_shell(['sed', '-i', 's/subvolumes/volumes/g', 'volumes/{0}'.format(auth_metadata_filename)]) + + # Verify that the auth metadata file stores the tenant ID that the + # auth ID belongs to, the auth ID's authorized access levels + # for different volumes, versioning details, etc. + expected_auth_metadata = { + "version": 2, + "compat_version": 1, + "dirty": False, + "tenant_id": "tenant", + "subvolumes": { + "groupid/volumeid": { + "dirty": False, + "access_level": "rw" + } + } + } + + # Induce partial auth update state by modifying the auth metadata file, + # and then run recovery procedure. This should also update 'volumes' key + # to 'subvolumes'. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + auth_metadata = vc._auth_metadata_get("{auth_id}") + auth_metadata['dirty'] = True + vc._auth_metadata_set("{auth_id}", auth_metadata) + vc.recover() + """.format( + group_id=group_id, + volume_id=volume_id, + auth_id=guestclient["auth_id"], + ))) + + auth_metadata = self._volume_client_python(volumeclient_mount, dedent(""" + import json + auth_metadata = vc._auth_metadata_get("{auth_id}") + print(json.dumps(auth_metadata)) + """.format( + auth_id=guestclient["auth_id"], + ))) + auth_metadata = json.loads(auth_metadata) + + self.assertGreaterEqual(auth_metadata["version"], expected_auth_metadata["version"]) + del expected_auth_metadata["version"] + del auth_metadata["version"] + self.assertEqual(expected_auth_metadata, auth_metadata) + + # Check that auth metadata file is cleaned up on removing + # auth ID's access to volumes 'volumeid'. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.deauthorize(vp, "{guest_entity}") + """.format( + group_id=group_id, + volume_id=volume_id, + guest_entity=guestclient["auth_id"] + ))) + self.assertNotIn(auth_metadata_filename, self.mounts[0].ls("volumes")) + + # Check that volume metadata file is cleaned up on volume deletion. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.delete_volume(vp) + """.format( + group_id=group_id, + volume_id=volume_id, + ))) + self.assertNotIn(vol_metadata_filename, self.mounts[0].ls("volumes")) + + def test_update_old_style_auth_metadata_to_new_during_authorize(self): + """ + From nautilus onwards 'volumes' created by ceph_volume_client were + renamed and used as CephFS subvolumes accessed via the ceph-mgr + interface. Hence it makes sense to store the subvolume data in + auth-metadata file with 'subvolumes' key instead of 'volumes' key. + This test validates the transparent update of 'volumes' key to + 'subvolumes' key in auth metadata file during authorize. + """ + volumeclient_mount = self.mounts[1] + volumeclient_mount.umount_wait() + + # Configure volumeclient_mount as the handle for driving volumeclient. + self._configure_vc_auth(volumeclient_mount, "manila") + + group_id = "groupid" + volume_id1 = "volumeid1" + volume_id2 = "volumeid2" + + auth_id = "guest" + guestclient_1 = { + "auth_id": auth_id, + "tenant_id": "tenant1", + } + + # Create a volume volumeid1. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + create_result = vc.create_volume(vp, 10*1024*1024) + print(create_result['mount_path']) + """.format( + group_id=group_id, + volume_id=volume_id1, + ))) + + # Create a volume volumeid2. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + create_result = vc.create_volume(vp, 10*1024*1024) + print(create_result['mount_path']) + """.format( + group_id=group_id, + volume_id=volume_id2, + ))) + + # Check that volume metadata file is created on volume creation. + vol_metadata_filename = "_{0}:{1}.meta".format(group_id, volume_id1) + self.assertIn(vol_metadata_filename, self.mounts[0].ls("volumes")) + vol_metadata_filename2 = "_{0}:{1}.meta".format(group_id, volume_id2) + self.assertIn(vol_metadata_filename2, self.mounts[0].ls("volumes")) + + # Authorize 'guestclient_1', using auth ID 'guest' and belonging to + # 'tenant1', with 'rw' access to the volume 'volumeid1'. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.authorize(vp, "{auth_id}", tenant_id="{tenant_id}") + """.format( + group_id=group_id, + volume_id=volume_id1, + auth_id=guestclient_1["auth_id"], + tenant_id=guestclient_1["tenant_id"] + ))) + + # Check that auth metadata file for auth ID 'guest', is + # created on authorizing 'guest' access to the volume. + auth_metadata_filename = "${0}.meta".format(guestclient_1["auth_id"]) + self.assertIn(auth_metadata_filename, self.mounts[0].ls("volumes")) + + # Replace 'subvolumes' to 'volumes', old style auth-metadata file + self.mounts[0].run_shell(['sed', '-i', 's/subvolumes/volumes/g', 'volumes/{0}'.format(auth_metadata_filename)]) + + # Authorize 'guestclient_1', using auth ID 'guest' and belonging to + # 'tenant1', with 'rw' access to the volume 'volumeid2'. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.authorize(vp, "{auth_id}", tenant_id="{tenant_id}") + """.format( + group_id=group_id, + volume_id=volume_id2, + auth_id=guestclient_1["auth_id"], + tenant_id=guestclient_1["tenant_id"] + ))) + + # Verify that the auth metadata file stores the tenant ID that the + # auth ID belongs to, the auth ID's authorized access levels + # for different volumes, versioning details, etc. + expected_auth_metadata = { + "version": 2, + "compat_version": 1, + "dirty": False, + "tenant_id": "tenant1", + "subvolumes": { + "groupid/volumeid1": { + "dirty": False, + "access_level": "rw" + }, + "groupid/volumeid2": { + "dirty": False, + "access_level": "rw" + } + } + } + + auth_metadata = self._volume_client_python(volumeclient_mount, dedent(""" + import json + auth_metadata = vc._auth_metadata_get("{auth_id}") + print(json.dumps(auth_metadata)) + """.format( + auth_id=guestclient_1["auth_id"], + ))) + auth_metadata = json.loads(auth_metadata) + + self.assertGreaterEqual(auth_metadata["version"], expected_auth_metadata["version"]) + del expected_auth_metadata["version"] + del auth_metadata["version"] + self.assertEqual(expected_auth_metadata, auth_metadata) + + # Check that auth metadata file is cleaned up on removing + # auth ID's access to volumes 'volumeid1' and 'volumeid2'. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.deauthorize(vp, "{guest_entity}") + """.format( + group_id=group_id, + volume_id=volume_id1, + guest_entity=guestclient_1["auth_id"] + ))) + + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.deauthorize(vp, "{guest_entity}") + """.format( + group_id=group_id, + volume_id=volume_id2, + guest_entity=guestclient_1["auth_id"] + ))) + self.assertNotIn(auth_metadata_filename, self.mounts[0].ls("volumes")) + + # Check that volume metadata file is cleaned up on volume deletion. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.delete_volume(vp) + """.format( + group_id=group_id, + volume_id=volume_id1, + ))) + self.assertNotIn(vol_metadata_filename, self.mounts[0].ls("volumes")) + + # Check that volume metadata file is cleaned up on volume deletion. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.delete_volume(vp) + """.format( + group_id=group_id, + volume_id=volume_id2, + ))) + self.assertNotIn(vol_metadata_filename2, self.mounts[0].ls("volumes")) + + def test_update_old_style_auth_metadata_to_new_during_deauthorize(self): + """ + From nautilus onwards 'volumes' created by ceph_volume_client were + renamed and used as CephFS subvolumes accessed via the ceph-mgr + interface. Hence it makes sense to store the subvolume data in + auth-metadata file with 'subvolumes' key instead of 'volumes' key. + This test validates the transparent update of 'volumes' key to + 'subvolumes' key in auth metadata file during de-authorize. + """ + volumeclient_mount = self.mounts[1] + volumeclient_mount.umount_wait() + + # Configure volumeclient_mount as the handle for driving volumeclient. + self._configure_vc_auth(volumeclient_mount, "manila") + + group_id = "groupid" + volume_id1 = "volumeid1" + volume_id2 = "volumeid2" + + auth_id = "guest" + guestclient_1 = { + "auth_id": auth_id, + "tenant_id": "tenant1", + } + + # Create a volume volumeid1. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + create_result = vc.create_volume(vp, 10*1024*1024) + print(create_result['mount_path']) + """.format( + group_id=group_id, + volume_id=volume_id1, + ))) + + # Create a volume volumeid2. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + create_result = vc.create_volume(vp, 10*1024*1024) + print(create_result['mount_path']) + """.format( + group_id=group_id, + volume_id=volume_id2, + ))) + + # Check that volume metadata file is created on volume creation. + vol_metadata_filename = "_{0}:{1}.meta".format(group_id, volume_id1) + self.assertIn(vol_metadata_filename, self.mounts[0].ls("volumes")) + vol_metadata_filename2 = "_{0}:{1}.meta".format(group_id, volume_id2) + self.assertIn(vol_metadata_filename2, self.mounts[0].ls("volumes")) + + # Authorize 'guestclient_1', using auth ID 'guest' and belonging to + # 'tenant1', with 'rw' access to the volume 'volumeid1'. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.authorize(vp, "{auth_id}", tenant_id="{tenant_id}") + """.format( + group_id=group_id, + volume_id=volume_id1, + auth_id=guestclient_1["auth_id"], + tenant_id=guestclient_1["tenant_id"] + ))) + + # Authorize 'guestclient_1', using auth ID 'guest' and belonging to + # 'tenant1', with 'rw' access to the volume 'volumeid2'. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.authorize(vp, "{auth_id}", tenant_id="{tenant_id}") + """.format( + group_id=group_id, + volume_id=volume_id2, + auth_id=guestclient_1["auth_id"], + tenant_id=guestclient_1["tenant_id"] + ))) + + # Check that auth metadata file for auth ID 'guest', is + # created on authorizing 'guest' access to the volume. + auth_metadata_filename = "${0}.meta".format(guestclient_1["auth_id"]) + self.assertIn(auth_metadata_filename, self.mounts[0].ls("volumes")) + + # Replace 'subvolumes' to 'volumes', old style auth-metadata file + self.mounts[0].run_shell(['sed', '-i', 's/subvolumes/volumes/g', 'volumes/{0}'.format(auth_metadata_filename)]) + + # Deauthorize 'guestclient_1' to access 'volumeid2'. This should update + # 'volumes' key to 'subvolumes' + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.deauthorize(vp, "{guest_entity}") + """.format( + group_id=group_id, + volume_id=volume_id2, + guest_entity=guestclient_1["auth_id"], + ))) + + # Verify that the auth metadata file stores the tenant ID that the + # auth ID belongs to, the auth ID's authorized access levels + # for different volumes, versioning details, etc. + expected_auth_metadata = { + "version": 2, + "compat_version": 1, + "dirty": False, + "tenant_id": "tenant1", + "subvolumes": { + "groupid/volumeid1": { + "dirty": False, + "access_level": "rw" + } + } + } + + auth_metadata = self._volume_client_python(volumeclient_mount, dedent(""" + import json + auth_metadata = vc._auth_metadata_get("{auth_id}") + print(json.dumps(auth_metadata)) + """.format( + auth_id=guestclient_1["auth_id"], + ))) + auth_metadata = json.loads(auth_metadata) + + self.assertGreaterEqual(auth_metadata["version"], expected_auth_metadata["version"]) + del expected_auth_metadata["version"] + del auth_metadata["version"] + self.assertEqual(expected_auth_metadata, auth_metadata) + + # Check that auth metadata file is cleaned up on removing + # auth ID's access to volumes 'volumeid1' and 'volumeid2' + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.deauthorize(vp, "{guest_entity}") + """.format( + group_id=group_id, + volume_id=volume_id1, + guest_entity=guestclient_1["auth_id"] + ))) + self.assertNotIn(auth_metadata_filename, self.mounts[0].ls("volumes")) + + # Check that volume metadata file is cleaned up on 'volumeid1' deletion. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.delete_volume(vp) + """.format( + group_id=group_id, + volume_id=volume_id1, + ))) + self.assertNotIn(vol_metadata_filename, self.mounts[0].ls("volumes")) + + # Check that volume metadata file is cleaned up on 'volumeid2' deletion. + self._volume_client_python(volumeclient_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.delete_volume(vp) + """.format( + group_id=group_id, + volume_id=volume_id2, + ))) + self.assertNotIn(vol_metadata_filename2, self.mounts[0].ls("volumes")) + def test_put_object(self): vc_mount = self.mounts[1] vc_mount.umount_wait() diff --git a/src/pybind/ceph_volume_client.py b/src/pybind/ceph_volume_client.py index 3d898556dbf43..393d0033acdbe 100644 --- a/src/pybind/ceph_volume_client.py +++ b/src/pybind/ceph_volume_client.py @@ -335,7 +335,10 @@ class CephFSVolumeClient(object): for auth_id in auth_ids: with self._auth_lock(auth_id): auth_meta = self._auth_metadata_get(auth_id) - if not auth_meta or not auth_meta['volumes']: + # Update 'volumes' key (old style auth metadata file) to 'subvolumes' key + if auth_meta and 'volumes' in auth_meta: + auth_meta['subvolumes'] = auth_meta.pop('volumes') + if not auth_meta or not auth_meta['subvolumes']: # Clean up auth meta file self.fs.unlink(self._auth_metadata_path(auth_id)) continue @@ -351,7 +354,7 @@ class CephFSVolumeClient(object): """ remove_volumes = [] - for volume, volume_data in auth_meta['volumes'].items(): + for volume, volume_data in auth_meta['subvolumes'].items(): if not volume_data['dirty']: continue @@ -395,13 +398,13 @@ class CephFSVolumeClient(object): # Recovered from partial auth updates for the auth ID's access # to a volume. - auth_meta['volumes'][volume]['dirty'] = False + auth_meta['subvolumes'][volume]['dirty'] = False self._auth_metadata_set(auth_id, auth_meta) for volume in remove_volumes: - del auth_meta['volumes'][volume] + del auth_meta['subvolumes'][volume] - if not auth_meta['volumes']: + if not auth_meta['subvolumes']: # Clean up auth meta file self.fs.unlink(self._auth_metadata_path(auth_id)) return @@ -1003,9 +1006,9 @@ class CephFSVolumeClient(object): # Existing meta, or None, to be updated auth_meta = self._auth_metadata_get(auth_id) - # volume data to be inserted + # subvolume data to be inserted volume_path_str = str(volume_path) - volume = { + subvolume = { volume_path_str : { # The access level at which the auth_id is authorized to # access the volume. @@ -1028,9 +1031,13 @@ class CephFSVolumeClient(object): auth_meta = { 'dirty': True, 'tenant_id': tenant_id.__str__() if tenant_id else None, - 'volumes': volume + 'subvolumes': subvolume } else: + # Update 'volumes' key (old style auth metadata file) to 'subvolumes' key + if 'volumes' in auth_meta: + auth_meta['subvolumes'] = auth_meta.pop('volumes') + # Disallow tenants to share auth IDs if auth_meta['tenant_id'].__str__() != tenant_id.__str__(): msg = "auth ID: {0} is already in use".format(auth_id) @@ -1044,7 +1051,7 @@ class CephFSVolumeClient(object): tenant=auth_meta['tenant_id'] )) auth_meta['dirty'] = True - auth_meta['volumes'].update(volume) + auth_meta['subvolumes'].update(subvolume) self._auth_metadata_set(auth_id, auth_meta) @@ -1052,7 +1059,7 @@ class CephFSVolumeClient(object): key = self._authorize_volume(volume_path, auth_id, readonly, existing_caps) auth_meta['dirty'] = False - auth_meta['volumes'][volume_path_str]['dirty'] = False + auth_meta['subvolumes'][volume_path_str]['dirty'] = False self._auth_metadata_set(auth_id, auth_meta) if tenant_id: @@ -1209,8 +1216,12 @@ class CephFSVolumeClient(object): # Existing meta, or None, to be updated auth_meta = self._auth_metadata_get(auth_id) + # Update 'volumes' key (old style auth metadata file) to 'subvolumes' key + if auth_meta and 'volumes' in auth_meta: + auth_meta['subvolumes'] = auth_meta.pop('volumes') + volume_path_str = str(volume_path) - if (auth_meta is None) or (not auth_meta['volumes']): + if (auth_meta is None) or (not auth_meta['subvolumes']): log.warning("deauthorized called for already-removed auth" "ID '{auth_id}' for volume ID '{volume}'".format( auth_id=auth_id, volume=volume_path.volume_id @@ -1219,7 +1230,7 @@ class CephFSVolumeClient(object): self.fs.unlink(self._auth_metadata_path(auth_id)) return - if volume_path_str not in auth_meta['volumes']: + if volume_path_str not in auth_meta['subvolumes']: log.warning("deauthorized called for already-removed auth" "ID '{auth_id}' for volume ID '{volume}'".format( auth_id=auth_id, volume=volume_path.volume_id @@ -1230,16 +1241,16 @@ class CephFSVolumeClient(object): self._recover_auth_meta(auth_id, auth_meta) auth_meta['dirty'] = True - auth_meta['volumes'][volume_path_str]['dirty'] = True + auth_meta['subvolumes'][volume_path_str]['dirty'] = True self._auth_metadata_set(auth_id, auth_meta) self._deauthorize_volume(volume_path, auth_id) # Filter out the volume we're deauthorizing - del auth_meta['volumes'][volume_path_str] + del auth_meta['subvolumes'][volume_path_str] # Clean up auth meta file - if not auth_meta['volumes']: + if not auth_meta['subvolumes']: self.fs.unlink(self._auth_metadata_path(auth_id)) return -- 2.39.5