From 300f6435095001258c11300937230c399802f46e Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Sat, 1 Mar 2025 12:35:51 +0530 Subject: [PATCH] rgw/cloudrestore: Add Restore support from Glacier/Tape cloud endpoints Unlike regular S3 cloud services, restoring objects from S3/Tape or AWS Glacier services would require special handling. We need to first restore the object using Glacier RestoreObject API and then download it using GET. https://docs.aws.amazon.com/cli/latest/reference/s3api/restore-object.html This PR adds that support for "Expedited" tier retrieval type. That means the restore would be quick and the object can be downloaded soon. TODO: "Standard" tier-type support. Need to handle the case where in restore from cloud endpoint could take a longer time and need to be monitored periodically in the background. Signed-off-by: Soumya Koduri --- src/rgw/driver/rados/.rgw_lc_tier.cc.swm | Bin 69632 -> 0 bytes src/rgw/driver/rados/rgw_lc_tier.cc | 177 +++++++++++++++++++++++ src/rgw/driver/rados/rgw_lc_tier.h | 15 ++ src/rgw/driver/rados/rgw_rados.cc | 19 ++- src/rgw/rgw_common.cc | 1 + src/rgw/rgw_common.h | 2 + src/rgw/rgw_rest_s3.cc | 5 +- src/rgw/rgw_sal.h | 2 +- 8 files changed, 210 insertions(+), 11 deletions(-) delete mode 100644 src/rgw/driver/rados/.rgw_lc_tier.cc.swm diff --git a/src/rgw/driver/rados/.rgw_lc_tier.cc.swm b/src/rgw/driver/rados/.rgw_lc_tier.cc.swm deleted file mode 100644 index 990e1367cbe0208b9ece2aa1fd6d5777ffd4e816..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69632 zcmeI533yyrdG|#rl!UcxVb6^U%1D;RN!VpuMUj<6U|U9#NrFj+(P(5%JepBvMqV%> zkc5&JXxT{!P}Z^pXv$uemXy+#u(f<8?4=OkV^7$^QlQ`O|DJR1GK*|yfv-LoKL?Nhy#xQy*Vm}mTe^I>Pqkhu zR+s9{R^GlX)R&8uT9-smQ-&M&*Bg5`mdYzj8=H;N#>Srg8y6atedWf+MscCu)P+K| zRA^PojeM!pC71O}>n*U}0!Q5ftF6V6N8P#ap{JZ|L;a8XPl)bu_gzPw&3c0M7FchA z^%huff%O(xZ-Mm|SZ{FchA|G6#DTDfcAg%tnyd=>t}|GiDy-(&sn^ZeiEwcWp? z|9!6i`^&cbck;i>{_k1-1>rq+_Sc{7|9-IT{$2d_XZpWq_yULYclFmV^?yInb|2G* zeOCS7UAJ{#Li*qL*Yp1GHEs8A=dVxtzdvice|vwu?*Bf@XBhJT1Al$L|NDlv`*-ly zL;F3*zc8df#^2xL|30R~?4uRd^Vc=NsB@7my2cH9P2bY6Oz+P}Bcmg;H+!Oo)<@zh&U%=bJ z72u`dC14S32d9AhfZKrYqOAWr_*d`=a0Pf7coKLb7y`dU;r}A|0C+FB61)k#5!Asl zm;(<7cL(3afbf68=fP*ewcr)t3UChC0k(n1fuEq;UkTm>UIw; z!5hIP;9RgBYy)=yH_++VgZ}`Z1hRL06kG-V3j7&3AM6D81UEt-*MTpCPlKz$2f+ux zRp4#lt-$E0udmrEwkoBlQfoyktF6N7O0`~GDCp{HGgqqDnyqMSeYLf-QC_SZ*x9J> zt1Ofo(a^%m%5X#k$mnRZSREbRYJnkKw!g!DeHZjaQKQ^iZPcQZHudpC7vf7JlT*{1 zbTKMdo8{<`-E&WLJfmb$w0Uz>&hK9;wsM2|zEG|$RF#81RSHvBppZE$)yXH>BmIlT zO0~QYwdy=wsxOoySE;Bzf8J=+-=~xpM5SV@v=rs$R~Hw{jnUC^qfu`RMCCzWrpPB6 z8FP8$RtCdci>u_k5^ag*tNGG=c~7O5^Y>bfgOCOjW)+msf4xFYPE2%Qm!rxN7T;Vpr^K|@e6y(twQr)vsGR$P^%nY`7<1?)a%vZ zs9qt>JYST@N~@84Q4w5?s}=fp-dfV?<8n~$3h@~kYwj!M2Ki$dtCWr+>{;^XLD#5y z{m2+s^o;HfSRvZ(t6W`(;bhmOZQba5adKv6dS*1*QEx@XXhx-)ZPgp)d^Az4)wnoc zj#`aktyyVR>NVHhrD}b3A>W^&6SeLMRw>^F3GF7CMnm=lrCvB#D=t?`g;KHE+T88Y zp|PB!b?Xt09@i@n`yrYjG?Y52QQWWQiH4GOQI0l5{TuuFUj()Fj*)&^rb*jKY(T;{&3u4I(dPmFpkrALRAC zE?FukBbq2WK^K>b`^p6yE8%u-(2Zg1_;k8_d3mLEkl{EmkW^c5+hWrE;(C3zdyqv}cXaoSB-NEKF=0 zpPemC?wFX~IysY5!}KEX!M+QUcG)_q1clS4c1+El0l)N2)brbNtGGv8EPHjiRaq`~ zUKYVMD^KZi#}6FG5q)0%#j|tcGjq9<du!IIc!pO?=9vCa&bJFRlE|)&jG*8?q{=x<*l- zqoeSgD)F6+e=DhOtitLZ@vuTmqu1J67t=Y!McmX!o~uylT9lv`G+KzFR#@FhLO@GNGU(LX^)n&VsePJ>~ z+ca(SGeo=E6vq_x64>ELQi_8=)6QV3c3#PhCbfEIb97D|^)*E)!v%c~)0T_2uPyt@C>KJX}3HpAHe{9obh) znTI#K*KI_iuhsKQsV3_8cTTG}mWwU;xzZ)|+6>vJp59!c{b7X5>1&~IW2nI!qkWa- z(Wuqj@~Eh}f6FORa~W!C73ZVH)|S%Bik=u58i|&f2Sz70Y}f>+n;+R~k`1=d?&y#>}= zV7&$Yr?h}HZ>*fM>GKD5Cz!Sp%dHlJ7D}xHd9%c4O}L3YJ~7zJj37%E`E4k*o=BT#2dHo+BP?}b9`p5uzhMr zVdwPh?9{2-CJT0DcIt_fQkeC+o>^=@Y!<#Ku}a)CE1D*BYp-WAD!(;BDp5VZj3Fhj z%PFFvWxaPu-PwzTPVqD}?dT+DL=qRLE(5ga1?zC785OItk{`6rn zhkO=`_|W97YJ%6TygvedM*f$S3{ zXs3k(DPaVGy=A!`EyL%i>BVyQMoye>a;-#1MK{v_?=PP=kp7?d)5iPJ^Bdq4a0hS= zI{p>lWneEj4*Uw;{=cN(gSUYOI3Mf=_W*YXH=^Ia30wv)1($$}K>^$z{1mo6L^<9QPcVjUm(5eE5pM4Y(r z6&nezHdaMk9kqNDVW~g~>Z^@X*+uX4BkRt&;bd2dW+^s`%gwgvm1-XE8mV*5X5PlQ zl3~1a;$e##MO6k&nIU`~nY9^fMlv}L)N^k8Yk>Oy;>~s)MM-O3@C&z{4C5gu7Yk%DU*c5rr)xaQt}p8 zSiL3U<632PdBxh~w9wPh(BhzvN;;&jrn3*=#VS=&4PlHTXFkIEit1IYZVqbCSX6ND z$=*&m6nVlGb<|3fv*(#pbP)%&tHlKfA--4QS7h?gb2L}`Gf6R23g3%?t>_X2Hl)7= zQoD9@zdR*`^j0*UJ$aKwe>BSIsYv4c@flh{M;LF56~*_^8Dx|ynE)e3d z(7#F2#$j{uYK}?3 zg<2zKGR@^hREIaW5LRJMb~i)r+%+e5AR#V0P1@9-+mD;M%%D~*h6O>aSP(>*;~lr~ z(+Gb8QHrzs$VC%J>55bf1anEdM(fN(QrZ{uyik&wd%yc-BOleVSZb{nt1!0WLNpMa zsS--Bg&IO>ah)b6om?D)Vv|R!Ag8^g8T0X{ugN44UP6Q`k1%9X*;~m7f6U}=8CjM} zp3aV-0wPCh3zj8+X@ulqZW8#F{5C7e2m_XCP{;6ktE3ZUZ2t2VjnbbWBMjP$e2YP4 zv|z2gUtt>CCMFaFIctM%Cy}vPN`a13@gsghklD^ip8`4duta+-6bK#N+$GB~-}pB5 zxzfciv2l|8a`qn=T4P0aJ>FJgS6J0QDQ&5Q$s*23LQSv2C8@_CZwN~5iVZ{Yp~bxt z9IKc;uyR;DQ+XdQc4(^VEgrfrYY57f?l<$X-DLk#xfZQ9Q9h(~wTf7M_7xkI;(QhJ zMULo?yo5B~rx^iul`>;A%jM-JrVR>02hUA!ogR(0x&RSN5s1A@&iM#t8a3F-5cqrWafWI*J2C!2-pE03r4^l zzz@*(>)=U1as9i%eZV)+_g?@m1P8$ZPy-|2cHq|X(1 z46J>8+pPatZ-Mm|SZ{FchATcrgMG1jfNe%;l&)mCfbvaOI9{1E)HQLbBU*R8gh zfCe#;!&;BMy1j0-Ew5W`O@B@%opq}%i~drsebd8S;>yB^AZMETFi|e)$|y;QacS${ZV@iAvqKJyD*gs@lpw7{ zEHcK?vFzhPS*hB<7iWGBNB>b}AsmH1%mAz&BEG;%~ zvHwoAsI{SNGD3W+)%t!3N=n|0_7hL*cKE>a+i3YwNC<4|7WB1osGZ2XQ?i#@)e(uK zw(=22Nt^rHX=+_FFecuIMy;Rnfl?@aT-ea985A?X3f4w*So7&o&3rt65ex1#?eZ_ z?R}G(UR5G2~0#kyP58I&;Gpc|Rq%Jtpa=B+-nwryt{FmBlK%QUc zwvGh|`{K%$*)jkD&J$N;pIb;W(0IE!QbC6C9_?UvyyWP|dB2VBAOKh#UjJZc_(Ylm z&|Ui6Vk}_mEY!ioGh6qr&qJop)j_2D%M_hr$XxX}pHDjoy|LgM;um6Yr=|7E* z|2m+304@XvK^ZK7ad0BIEBGb){@1~E;6va~zz*;L@GbQGuL1f0KLcJ0UIH!ziUHUI zjt8;<{1p5ITnF}nMKBM}0rvv`jK2Ry@EUL-(Eb45MYsO~xE6c{ybM&pUBIu=>$R`n z&%q7ga?k{)g5RLqe;<4sd5z)QhPz^UL5z*o@u{{wskyaikdE(6a1JHU1@1hhs#>jSKx<85)=q~SK>N-eF* zE@5$dS3#74%cD5O!87fvuwwpyI6Xd5zC)~A#I90>><%irB0bY~P@x2R^r5G)PY}DR zIJG~u1B=}-LxHBEki?DuxV*k<8ZPmEK;HPNC*$Ypd8ABjXcac}Nd*kG@4Dly71{77 zYpv97I;os(-I~QMr84ymsj#?V$z3r6n9m+zZyi6Esq&1zs%|tzd zlMRwivQ>#HQ$}d$r7u|N8LbOtS>4+9`O6k9`Q`T7j0TdI+V~*WEELmr@w5`!GC0wtW-SXBT@N|NmU+N)zO~J1;+vUNx}Yrm*ysx~ zl^@Ul(~W2yrs`e4Gb36kO3Yx_4QXALi>({dQhPjz^|i5&UUNVfcH)@2E#C`M0Sk-V z=ugiEy{1L+VSR2fCHmH0Toy)HY|e3m-L&>@tIYatf~9P7uDFK`P5#u}9t*|H3s`sK zRjjv~fox4v+%`XR+#;qm)zO)_QWNjttZEr+7*YRjCUX*OFXv%mb4%9BIk%a6y_1~i zEca71*=?uK>TH-Cqx0 z46F~@%A;00r(|vM7cEHY-4jrS6wQ5=Vk()NQ&Z}Sn@-zX z$#tc@0}~AV#4REZcAA^q({2qa4IGyH^@--gXCUP7*4&BL#nY2}Ci6fDI&inztKjbH zRT=fp7LCUutQnW$1!KcbV$e%$y&Va?VagRKHcvnp9XEK5v#M~ELoQe~2GxY#WkOHc zw`e3-zAl3wvo>8E>4iett`V^3w))Xv2nZ!>@yO3?LMEiw+$TQH7@{vOi@adGD4; zM;N#c^$Rnt^sLDoj5a2G;I}CX=bE{HY^-J2lV!@78``JOto;`Yg><;kgV68U&vaT1#6YLb(jW@cpK*y&# z&(O)v4au)g>m6|djs*tN@eEO1)VF$KrZzd!^^`F z@h*Q#*cRQOOC4;v&99qORCdGoSNr51%0hrauvQ0ID4Fp?dnAUh*|4yAbkyE!bAs8r zdTX(|UK(GrV!0B{Xob;wX?=i`oYpZe4e0`-p?TYl?8#2)h?112?WGw|za;RVTzHuWKh!L=2s<6T6to{9|sn{9MotWyrXO3Pn{r@p3|NkDo>!bDm-;k~k{tUbuUH?VkdEgRo5qJiW z4dAEf_}>8^0B;03)2{`Z;B4>%^!)FGFM+eb8DJ}z0N11E{{U!h|MS2zK|gp9cp$hd zxHGsD_$d1RJHhM09C!qXfa3grh0d?@`~CsE8@vmg0Uil%547I@D)jyeSOQM~*P`El z8oUcU59|Xs$d?bo8GPEyZxP%M{0LqDd*Hj^tKe1O0GI?fqVFrNU+3?A2|OL#75oZa z|3(n}`QPCCb>M5@DPS)+4{QO)fqMh_^Ir*G0TkQ+2yh4R2SEP&AA_6oiTI?rgXdj= z8>F>sI0Jq|M>jDA+tAFCY-Xes{GOMNRpU3DT8tX2waBbajCXdQU{2L}S=yk}J#4|B zE*Bp==XWTHM@83HFsbGUV}-BzQu|#ywS|{sA>3`h$yq9?y+E@0`hgAaai+m?D}7eZ z7;=F>sfkEbaj1CSpgnv}cGg*HJ5%2pVz=J;M~P4*><0`k)!ywJ+9Z<6T)Tfd#MHGF z5QR=Vv=m%fY^g@tb(h`%C^gGVDoY$s}W6&Bs|EVf+!K=xNcMf!cIT z-PwPt1ughpC8zfjhH>h_v+Jwj*~tSOKInFzq{V|sW5j)=)x1e@^r9GBk;R`Q zilnD#;he}e6FXM1-|a?`@t`xqQ?@y}h0!|_cWk1k^Y11!kTDr%YPMgzwYH7SR<)Nz zCc%8|{UcoygeV>4`@_W5JF5Tf>>k;*j;_FMW7l_HoS>6A#zH59$+l^2Im3%2$+XEu zU1!&jKH~{S<8iIeQ8?t{B1-jZ+wn2JXXM+@)@0b3=n@~_+P-PquvoJnyx}QWbev0P zBZ%{-qE1~HYA-4Op`NHF9Aox-tBtyePwdl=8?#@n@zPZD^7^BWgrB%oc7?P#pzN6S zjD_aguD{wpkBL+KP!pB@j>*9m%Mk3YnF_#5DXX20?Zl1rQK*^54o7x%lo)Cip6oEaviMrb=J)hshp6z>FEUpS#YAo9RW?Z!HjxCg#(*b%> zQt!9vkb!K58h_a$`B9j?2%UG7J?SJHKTBwbghHnJsLh}dw^F+4)@rq0Tz0XnQ?WEM55Vbx!Q3;F8n38$E*|WQc+t*PZIpro&NO z`hbWnyZoYd6&PJYEf+c5!d5dXyaByh^xfL z(QN*@mG)Ju4Dq#&0~J$Gj)e;7={WYR$5@d8B(%dCLxq4xbaIw)WRDGC;8WuXv!XzK zzP?MWtnv>MC&{LF$xe1mAKGp)HR#N|?nWGf^(`D^*&~>ca`qxEXAWiwj(fnFHWK@x zqd!^Z;KR5ilz4bdl&dCa5gv#oB_U#zOQ}% zwD$iq;GIA=f%CxGU;>;96c_L%@I~+m@G)==I3Mf)iV@J+fML(SuV4#!IyfIJgDQ9& zI1$_i^nrI`19%5`J$Nn9T7U`|2H(R5@K&IG0Nw;@U@zDJZbbLL23!W73^stDqW}LR z(EfiD;2z+|=>M++uLQ3E3*c1n81NAAU)TWt348>+7!<%T_$B)PHQ*|6As7R{M)&_1 zP`v*$z!EqG{2cxMYVdyW9`G*kPVh=_F_1lA3G4w2U>M}UkJ0_#2Nc(@n0~GGSD$^p zr!64Titsb$=5`h)rgzL?N3{GGs z`ONe-lDJQjFnv3la`LR5Q!Mz7)3l}Zx3g(NF}IEHIDOan=~|S;CwZH%g>J_QLz=m> zclzhuo4O^`GfL>WAA%!c+XyFG`M;-J<5ZH81-55?`2BzT#HykpP8P9KfnjB>{o-cY zGnlxXeHb^&)dg4`=336JWp+wIqF7h*T-#Ah96N_WVy zOvfhO|54!byZbW@qOFVdp82YsSg?1Wj~24&Nit-a^IS~*rWs1hD><1zhC5nZ4L2C! zEnwn4L^bsB^WyGB4hJ`LD5}rmVczFbHuVyAId2uciN;;l1Sbiuz_~Fk?E*2Db5F*n zE2Q1}$^+K`+n#bLlx3B^+mTcH@s9kOwG{-^)?zMG7vxqZ_IlgZ!QrU?A^D-E&>voD z9hynJJN-iC!@=+yB3GaI=gHo?9xETIM08x-4cjN@#tS=VCQq9>OT?m&6RBN>)@i;+ zBX@YaPPli4@0Q6Rfd<}f?Q>h>*%7?f>7LKhOf4pwuEh-v6o{oL>zOt_W*{sahFDC1 z)$=k3nM5O_o~XH1cUyJa;0ilPhry-2N+NrYr~S;+M81)U+SxZ)dPKUm+Y&KM%v2c` zBdXCYg23&AvUeXFp}2Q58r*nrB3MS<83;pUu+K9bTLEl-Mc*E|bh1RBOjgLkHN@UwZX*%Mn7Al;m(i#cj ziv8k$h*q-?J<~m+eWmSOsnIBeSi1YE8;8kL&&Oo8X8C-J5SO7d7MeP!2r-wlV%#al zeat&?MzeF%Gt2>niEZPvvysgN{b!AgQ@L5EIgU)|D0iz+I^9`?ndwuf=Vl9Ytc2+BG{lQ`kPfW9l@vYj;x|38qaO7};K|RTlBpcBBeyc5=r= zm=^mdbm}%oEKk<3Vw#4h$u#wQI#*dI=U&il;?#h&DN7@Q^S?Odp<|4?E%cc_ zyUn0HotZ6d3Po2Yr~WSWeEg!XEF%0!Fx$^l2h<57|F=FU6eH{XeA zL{lS3p|TKfwcx(Qr>9UUW0@|2cxG{--i*Db?y@+bgEF1P#Oebjpb0% zRPkh>gk2Rom%P`0MiS?Ik+xc>565`8ke{w}B7(#MR^>dA;_K+Iq+a0{a>*Fj|aC0A4ZSA96SZ=1;>FeNnZ!= z0GES3UQ`wD0l?WetvHTd%y$0&(PC90p14w5Woe>34P9HI3l0E;igP>LO+Rr}=tq2;i*VrotfA zqm^Ooo=F+@+0)Aor1P@R^pq=UC93^Q#q6M<~p%RDD^L%jdW<%Lho|JChT$;XDF}*7%-jL?35Pu>! zCk-pNtE9HWFV^IMEeVCrr;I$+#h*ht62bRy?i)L+IOV@dQ`ed<9B5zT1Fg1J->-v> zwSApGE9&&gImtU0MEjR2)iMrls~pjF3RAUXPf2UV?6^Ud@U(SF5xbmRXt7rXTs|3b znYz1ydzjAX*k0nd;a59MnYt5*jg{XSh1^UBAKeUb9O921mffXi>GJr*>|V}5JH$yw zAyf&eyomob#wlh1vp9{W*|B;gy`)kHCHhg0I074XvIt*tDX#0}ZwYD^_K@Yb zE;JF6uo*-3YQagw)#VVSz1+BOo8Q-^Jsodu{8dz!lF`;>IoBX<|C?YG?_wW>Hg}}t zu$_GRzf98eBmIbD9B!txr`Z~N9_||qfNg!_e#Yi*Jrg@M&1su9WGAWP1nJw^gu#56 z*dFz+IAQ9x(rNwwFg)jT(mT=r%ly&jYV`aMf!Ba%g2#d>@E-Jitp(6|e~$!O_x}+v z2Tlc>!6xuW;FIY4^68hrx6tc91D+3#1GfXeMu*oPe%}D;ef$*PzYjbdoD5C^e+a&g zUjISxW^fUB2G|C)-v2`M`>=oi9N)E%-*3?8e-3^Ezz~spy=1ruxL?EP4ldQe5x)b4}^&+r_m1kHV$Bc&?yhu5y&ti&;)?r%SqijzWQz zT5jTI4_(5QvzPTaqk*P^tq|8Fz?nHdO9BeFKY>cg&)WjYo6Fw%39ZHWVuoX}a#P>&&jo5w0MTgU-f-A>@`e@gyAB(T4jUzf8u6XFRrE z$?Ypv!+aingQmBpHjZ$0w#PDIDCvN#W`PO7!cbI| zWTJ?pP&yvYM4|L4)qFKiV3`73Qmu}NP8YavA>RXxm&J0U$_gj%QVTD1^Ui=$#Iuf{ z%iP?P$O1zl&L!QIp{Mn59P8p|uhB4zsc*UOdbOigyA=j-3d}U03SraC8efk2s!)7a}8AP~Es>-;cP4iO*cpWLqOwM%-1K3``l{sdS^&K_7*TAA{?xRlVM~KmWyh}V zg~=T|r>BVMoSHUcAM)1R%+%zp0=vQh3^`zi3716OsdX*1evPBq{fRnW9&EFMIQI!_ zyq(E3ZA1+Y;1Ki9LtfhHa5J0vXcn(gw5LvGMPRIw#6MssfXKE~uZiz*9lv zE6t7CrzjuI*11Y}gWm(enY{h0b_aC^_H$00TN^{G)EJg+)f}W3io_t9{oEvo=~Ex8 zV5-@vCr-+|%Sr^<)yl0t^#`^Z*Ps5qW~kAPKs=pf)r)kps~=1j?U)U$YrJ2A&_S2E zGmSf{QCDVZ<_P;cWSX6o!n$`YrU*_V=jd0>ZHK9$jdZXh2V+LYB^1Tr=u!a?M?#|h zp$!`{HgRe+ zG+3G!w@D?n9$F#!38FRKqziglh?Vx{m?6Xd|69>9rT;(ME0P~a*Vh?>OFTnSK_7(UJ_-pWL@Ivqc@F(CBa50z#j{#a^ zumxz(flq;Vf-As9;ORhT3lza)Kn|P;euKS1Hi^%Ie*#TV0tIj`kX=Ii72XeMAA);< zUqBaM1YZDGgR8*VUbfIExX{8I==2JG%`6czGH%uQnotDfC0&gRq$iX^~|!t??qgFO-}$F>_3T2^T|iXe1193 zGonA>;$j{yZ*JO#D0zZvTg!FD_YK%Bt4es%#gVx*{sz0~uE<+OjM+ggfrt*0RHA|V z>9#q54{n?AT#)y<-IbWv4mpE_JIbMe_)A)?r_xuolfO}4l+nO$@HKVA_pRO0ce@qN zmL!QRTbueU?z*E07HklnjrZkC_Gcq%a&ZxzI=nXXoASk__cRhnZ&RQ6ariYL!)vzg z;_u;h(gr>kH@Eqw@U38*N9A0cF-+Soa!k&`6=N-41<9 z+T%59u}11O>9-^=Q6H$3of4PsQ1;Wyt*xxs(B!A4D9yW2BBha_;eJ`4WxL2J3#QZu zZ6`#u5#6ff)>SOUV;4()KH_1a_e2^dvzT_Tmr9yt`>5pPiJ=u;vWM)|Z1S{FEr+<= z_A^RTDKnueMDBa}Fv-dG5YtdPco-{!R~@s|69~ofi})OgQN#FHoWix6vLzZ&95LBZ zYd~x<_1o-DZXLC?g7MVUCTK3JUe?7mE0D_CdSP#wuxAxC-L|1$m#pyWQkB3_MBt=w zu3SOv3j)s+KG#+`9-zFnoVHS|@gvB%r1sN{6G7FgNgI(TC(I^KjP~h%Ca{Y8%2GAb zCrR6-eWSKQDUpre356;Dal!NA6elJFTXmUYfVL-qi#H0?o|+p} z=y16wTkK?Yf%EsAxk94_<}zTrm4#pKO?t05Wt!o4nO8y)4?8#L!KWGJJEEU~BWm`e z&A1cGA=*3D$!T6N!BEaa_6R#3GMGy(jpYmzu#wMI6G?JT3>yr?6YB5*^AMpT=7yYv z>8UIP$43J`U8E7;UeF0F&=6gVqe@9n8H~4Jh3iI5YmeA)E>)V(LruwnNNx^?So{ce zzo8@l+h%~(S_SuAVWrjB++IPcm&JwYl`$u$)gf(aLoaHaaMCI>j2KBd$ zC&Z&ZW(d5ZqeWv8vAu}!0-e*8VpRM#NnIDaPjQLq(|gzSsC#9%POu>zMVXGH-`a+G zLW>&WYB%u>QHfT}$ri6|4w@MyMf}J*Opn@%ClV?8iNsE&P!Nwu*2{D*2290ubnN~= z#@}@^YQ#q({34r5IB88+oa@%EA*jJ|`IpP->d;`y8`qF;*U$IZ(ulQVcbTvc194-h z5SX%9vaHpV+SwNVwoCH+`=)3ZDh<0xtp=gGHb-1h)Xuhe%SN zh4Nw%%_)dRx|A}s>nE{U2Ul!I@6|@7U!?CoCDsQ<;}hEoTH97QeP(*s&LhgDyo_0W zd|?4T+3cC!;opWC4IWQ@l(FZd?Dk@F4;=Cp@eob6$}rW)6Y>{?8rjE zQ9medqg1sGnC-WmMJ8WRX4U?LbQ$9y7}vq_JRuX|lVf(Mhs~|VA|@kS!W=uE+>~Su zMRt@w%kiFu9EI;?--jCqP@H>=%$(~nkNUp zURWC5@282Z*xx*vgiP$s*63)lx`%dcEiI3ZvUHxq(+T7+)mIMYmMZi&Gmy_AvtyMn zG27;{RbPuo{X)y2CM*Y&D z9O9Ylj$>?c@ku5@w-=#P=~v5OG_lUsV91!k-01L^_~S9 zx=NE7n~eRgB39mFf$QuVeuCFmJt4-0fv{-0T3#z=)jwt*F*`-jR8woM2SO$)duGV! zO-GOuN*w!C<9bVH1cx}>=TPF7_4+oiHwE=@DY;M3s+7ka;lwCpZx>_glx_3xTftaQ zR*^4PWpu0jmDZW%>}2m)QAoDU3M?t)xMrXe@lA<|L3@C<3K;l2cWDnHQU@k4ByueN z;RajqqtTSzmRd;Vb-gk|B%FSAC1Pii(yWjoSWlfv7G8ZF8PjaZCdzJEzF7mz4xsKV z&9`~V<9E(Fk0C0B#XTLBF`=zwn0mv^(tYHdIX<(4EedRI^a_mR2$kgA+mjX;%2c@IB0%YH*fx~HQ62F`hI67+QMz!sc8+f3b zXI;)LMOXWKdTBKddU?jCXLnCCAAwHui!mL?%#34JuNL!-Cu#l@gd7 zYjrX}kd`dm!2~9#%5A*GEaN5%v$J~knzP*G>v1eS?`8|*edcar7b?^6IG?(*cu*y& zw3-p3&{8WXcbcz#S3*)UR-v-Ot0=hkq6+i?(JJiIC?qp>;^A@{5gG9y@;S;Q19%MJGwt@^OqrNbiOSVQA0!btr>_rSSY+2E#&o^6N5wuN zE}>`DanspQ<#A_GO7`2Q2`AiX53KOhT%q>z?QasesyDPTqNR9rrcBHgTN{)|S@4Bv zSa#RgN@q=K_4oS&r|mlVC}AV%A#ME~O4-c^6nZEzt0g8H7^0T(nv=}vH8UoY6el?Y zx%tQx7Zk`g+g$ShABVm_jeaNn|57Rce5C(>9b5J~2rdB?FbN(3?gM^^e*ZP_VQ>X_4v_Ev zkHJYmdkGAJ`-A&|@1x&K|NkI(7f?)q_8536_+ub@z#oC*!GplB(EUFHUJeu!Fb~cJ z=YW3jGj#uJfz}$l30w|zp1{4qy}%FA_df#O3}h3~K7!|fzgI2QZ@J^wr4Tj2kKi$D>Kfd_(L(*HjPKLeiue*?50;nm<(Kx-0az)m1q zkzJtMCp9N}iNy^*H{XfD!v0bg@UBy=4A1flwoPRnlPrl9UxR&D%-6q z#o5Mu!wD8PQoNs8-!)*`9zphiX9Y1N`#|iBCV!&5)mf$lR1$& ze!~&CKHHixSR9?tlX3*dOJLMCE5s{T#bWhDS|K(gaL=yO(|A|S8sZ(Rp$P63?tcz6 zYUeq(e9Fy8rZFayD~bbZWAM20ip103-jrSaxoO4E6*uL;4JEey z>S_xxe7SH5e88_R&gxjTSr;~ErBld*l&AvNOucpI2 zttDGhF13Qd%y^n~*||ouslyVOT;K{6rjmA-XsT%B;f)*#NvFxtHE?&+drH?HODbzM zDB&0>;5C!h@5C&VdbHBWh3vZR9BGO!S{Pd?C1bXC-&t*4^Si$9rL%gYQ+ftYRO5Km zyLN1og~KN8ZOv;U&n>!MD9r5Ed#rx4{ra>FWrSiM7;(!`Mv(2A1l5dSqm&IO^lu)5 zla0kr6pA~v>)rHd4fdp7ZzI>v*JzU$y<+WxY6M$a31um}h^`+#wK<3^ITE+gQ^Y5` zAO?%lt+!gHt<+W?H`;ns+jmTPVWhlLFD)fy$KDgR5v49&7VNB|$);-7_U1U>g5_(Y zHLKpHy8==fDJg(pL>841E2|a7*7Egd8a{ru*8|!xvYi0kvm~{=(3^LZ_ZuZt=2zL$ z1d+M9QZAYLf$ovn(mMoHXN3PuYkD&sHP8l*w)SpzZhXhq@tLjmx{N$~n3TP2dZrh2 zyO|vq)NDeszk3pvWrq^5hlv6KMeVl8kKO zfZty{70_oHhiS4IFAhm5v!P@vLHqzTmRNYL4U%<~OlW$xV-DSC8Y=3){`#5Nn5CZf zvv{a#C{c&QC#g^gy|xPOmz&bIH&uhiexYCQ7nzITqspnvbOD$d)#p&(zUkB1myy+mwAcw zrP7obz$t~2Hj@jSTNuxlzDHa8sJ=N}H$?sLlMTW37(bXaecH zpGL<&1P+1&;52X>@HTY#E5M(D6)+EW0mb;A4o(8c1MT(yJ|G?cGO!394;~2~3_gy| z{yq@)2$X)`00poeoDQVt4}%Thm+0)j0P^{3-@r@2)4_#cH+U?#5xxDJ;A`NkK>Gdv z0iOnc4lV{4fjqbl9sWx29Pn(Q`2IfzW8gO6I`sIff%X+R1fBvmfl(lz|NX%)(d%_~ zU=?fxgW$i>Fe(SF9x$9=;wFkyY?8ofxfl=rEgQarXYf7FYXp$n02;1 z^)$t#Fa0A4mo%@2+e!?pq`TW|Qya2(Wag({NjVZEZrWMe|=kgD>{ z-UrsF)ZDuw+H9L(v=05%!!U8)p30GWZse%k7=A3w)xGH24I6Ld&AU72?R;*mr(?gt>3e5hUb_v&y>jI zW!g|x`y0HLZc{3s5jrIqW7z2C18Z9&DgsVaztlNVw4_BzyS5~1jnk48v6!^XtogYJ z5FYb2h-a%TpGsw#Ief~h>KdZ0#4cymJt?ilZDtygR_|K4B(;m8v_&UsJX&msNIJ&u zUOd-C3F24bgda{2r@KoYeSF(Bw;dS+w1# z6z8j~jLmUwU|zl-x|Uwmo^OeAh1aFLjJ8#$6m;+0^w#OoXe&ox)Z7vwv`*K1U2@Jx zr{T(J)+r~MH|y3fWU|vQF;YXhwXn_t;Rz*fEMu|jb48imJ8U|O`ZpqDX*#_{=ZX~9 z`CmCb_+|cW+;yc|WZ`k(hHEj_*vOZ-Ht8ZKlK4yUG}LAR>~v8(&!Ag3&Zx17k)vCK z$GhO@ZcvVlbz2d2^GYW^;YKa(!`XsIQ##)4if7DbWK4fpvp}c`*66@VobN`i7^iT! zmSnb*&`6M$bPii%7x4)SMteg`N7Ac0b|MH3quDdK-qY98HuqrUbmi#%vJP`h_kNX~ z*E?*@Gj*7@bK2*cl~9hZ63*@tw~!rW+0)m8u;uQqgl-hSUNwdS%hMgfGLhJr9sUn%U zS)!|1)K=VhIV_eon1v+`Evjp}vz)+eqzV!6*>io?kd~rwq>f)aV5pVKz$$zN+{{!5k BiNpW^ diff --git a/src/rgw/driver/rados/rgw_lc_tier.cc b/src/rgw/driver/rados/rgw_lc_tier.cc index 1b576a318495..58fb86b0af1f 100644 --- a/src/rgw/driver/rados/rgw_lc_tier.cc +++ b/src/rgw/driver/rados/rgw_lc_tier.cc @@ -252,6 +252,70 @@ static const struct generic_attr generic_attrs[] = { { "ETAG", RGW_ATTR_ETAG }, }; +/* Restore object from remote endpoint. + */ +int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx, + std::map& headers, + real_time* pset_mtime, std::string& etag, + uint64_t& accounted_size, rgw::sal::Attrs& attrs, + std::optional days, + RGWZoneGroupTierS3Glacier& glacier_params, + void* cb) { + RGWRESTConn::get_obj_params req_params; + std::string target_obj_name; + int ret = 0; + rgw_lc_obj_properties obj_properties(tier_ctx.o.meta.mtime, tier_ctx.o.meta.etag, + tier_ctx.o.versioned_epoch, tier_ctx.acl_mappings, + tier_ctx.target_storage_class); + + rgw_bucket dest_bucket; + dest_bucket.name = tier_ctx.target_bucket_name; + target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + + tier_ctx.obj->get_name(); + if (!tier_ctx.o.is_current()) { + target_obj_name += get_key_instance(tier_ctx.obj->get_key()); + } + + if (glacier_params.glacier_restore_tier_type != GlacierRestoreTierType::Expedited) { + //XXX: Supporting STANDARD tier type is still in WIP + ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: Only Expedited tier_type is supported " << dendl; + return -1; + } + + rgw_obj dest_obj(dest_bucket, rgw_obj_key(target_obj_name)); + + ret = cloud_tier_restore(tier_ctx.dpp, tier_ctx.conn, dest_obj, days, glacier_params); + + ldpp_dout(tier_ctx.dpp, 20) << __func__ << "Restoring object=" << dest_obj << "returned ret = " << ret << dendl; + + if (ret < 0 ) { + ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: failed to restore object=" << dest_obj << "; ret = " << ret << dendl; + return ret; + } + + // now send HEAD request and verify if restore is complete on glacier/tape endpoint + bool restore_in_progress = false; + do { + ret = rgw_cloud_tier_get_object(tier_ctx, true, headers, nullptr, etag, + accounted_size, attrs, nullptr); + + if (ret < 0) { + ldpp_dout(tier_ctx.dpp, 0) << __func__ << "ERROR: failed to fetch HEAD from cloud for obj=" << tier_ctx.obj << " , ret = " << ret << dendl; + return ret; + } + + restore_in_progress = is_restore_in_progress(tier_ctx.dpp, headers); + } while(restore_in_progress); + + // now do the actual GET + ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, pset_mtime, etag, + accounted_size, attrs, cb); + + ldpp_dout(tier_ctx.dpp, 20) << __func__ << "(): fetching object from cloud bucket:" << dest_bucket << ", object: " << target_obj_name << " returned ret:" << ret << dendl; + + return ret; +} + /* Read object or just head from remote endpoint. */ int rgw_cloud_tier_get_object(RGWLCCloudTierCtx& tier_ctx, bool head, @@ -367,6 +431,30 @@ static bool is_already_tiered(const DoutPrefixProvider *dpp, return 0; } +bool is_restore_in_progress(const DoutPrefixProvider *dpp, + std::map& headers) { + map attrs = headers; + + for (const auto& a : attrs) { + ldpp_dout(dpp, 20) << "GetCrf attr[" << a.first << "] = " << a.second < days, + RGWZoneGroupTierS3Glacier& glacier_params) { + rgw_http_param_pair params[] = {{"restore", nullptr}, {nullptr, nullptr}}; + // XXX: include versionId=VersionId in the params above + + stringstream ss; + XMLFormatter formatter; + int ret; + + bufferlist bl, out_bl; + string resource = obj_to_aws_path(dest_obj); + + const std::string tier_v = (glacier_params.glacier_restore_tier_type == GlacierRestoreTierType::Expedited) ? "Expedited" : "Standard"; + + struct RestoreRequest { + std::optional days; + std::optional tier; + + explicit RestoreRequest(std::optional _days, std::optional _tier) : days(_days), tier(_tier) {} + + void dump_xml(Formatter *f) const { + encode_xml("Days", days, f); + if (tier) { + f->open_object_section("GlacierJobParameters"); + encode_xml("Tier", tier, f); + f->close_section(); + }; + } + } req_enc(days, tier_v); + + struct RestoreResult { + std::string code; + + void decode_xml(XMLObj *obj) { + RGWXMLDecoder::decode_xml("Code", code, obj); + } + } result; + + req_enc.days = glacier_params.glacier_restore_days; + req_enc.tier = tier_v; + encode_xml("RestoreRequest", req_enc, &formatter); + + formatter.flush(ss); + bl.append(ss.str()); + + ret = dest_conn.send_resource(dpp, "POST", resource, params, nullptr, + out_bl, &bl, nullptr, null_yield); + + if (ret < 0) { + ldpp_dout(dpp, 0) << __func__ << "ERROR: failed to send Restore request to cloud for obj=" << dest_obj << " , ret = " << ret << dendl; + } else { + ldpp_dout(dpp, 0) << __func__ << "Sent Restore request to cloud for obj=" << dest_obj << " , ret = " << ret << dendl; + } + + if (out_bl.length() > 0) { + RGWXMLDecoder::XMLParser parser; + if (!parser.init()) { + ldpp_dout(dpp, 0) << "ERROR: failed to initialize xml parser for parsing restore request response from server" << dendl; + return -EIO; + } + + if (!parser.parse(out_bl.c_str(), out_bl.length(), 1)) { + string str(out_bl.c_str(), out_bl.length()); + ldpp_dout(dpp, 5) << "ERROR: failed to parse xml restore: " << str << dendl; + return -EIO; + } + + try { + RGWXMLDecoder::decode_xml("Error", result, &parser, true); + } catch (RGWXMLDecoder::err& err) { + string str(out_bl.c_str(), out_bl.length()); + ldpp_dout(dpp, 5) << "ERROR: unexpected xml: " << str << dendl; + return -EIO; + } + + ldpp_dout(dpp, 0) << "ERROR: Restore request received result : " << result.code << dendl; + if (result.code != "RestoreAlreadyInProgress") { + return -EIO; + } else { // treat as success + return 0; + } + + ldpp_dout(dpp, 0) << "ERROR: restore req failed with error: " << result.code << dendl; + } + + return ret; +} + static int cloud_tier_abort_multipart(const DoutPrefixProvider *dpp, RGWRESTConn& dest_conn, const rgw_obj& dest_obj, const std::string& upload_id) { diff --git a/src/rgw/driver/rados/rgw_lc_tier.h b/src/rgw/driver/rados/rgw_lc_tier.h index 44ebaddfd9c8..60006da89149 100644 --- a/src/rgw/driver/rados/rgw_lc_tier.h +++ b/src/rgw/driver/rados/rgw_lc_tier.h @@ -57,3 +57,18 @@ int rgw_cloud_tier_get_object(RGWLCCloudTierCtx& tier_ctx, bool head, real_time* pset_mtime, std::string& etag, uint64_t& accounted_size, rgw::sal::Attrs& attrs, void* cb); +int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx, + std::map& headers, + real_time* pset_mtime, std::string& etag, + uint64_t& accounted_size, rgw::sal::Attrs& attrs, + std::optional days, + RGWZoneGroupTierS3Glacier& glacier_params, + void* cb); + +int cloud_tier_restore(const DoutPrefixProvider *dpp, + RGWRESTConn& dest_conn, const rgw_obj& dest_obj, + std::optional days, + RGWZoneGroupTierS3Glacier& glacier_params); + +bool is_restore_in_progress(const DoutPrefixProvider *dpp, + std::map& headers); diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 94861390c3d6..f34667901ff8 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -5290,14 +5290,8 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx, dest_placement.storage_class = tier_ctx.restore_storage_class; RGWRadosPutObj cb(dpp, cct, plugin, compressor, &processor, progress_cb, progress_data, [&](map obj_attrs) { - // XXX: do we need filter() lke in fetch_remote_obj() cb + // XXX: do we need filter() like in fetch_remote_obj() cb dest_placement.inherit_from(dest_bucket_info.placement_rule); - /* For now we always restore to STANDARD storage-class. - * Later we will add support to take restore-target-storage-class - * for permanent restore - */ - // dest_placement.storage_class = RGW_STORAGE_CLASS_STANDARD; - processor.set_tail_placement(dest_placement); ret = processor.prepare(rctx.y); @@ -5326,9 +5320,18 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx, real_time set_mtime; std::map headers; ldpp_dout(dpp, 20) << "Fetching from cloud, object:" << dest_obj << dendl; - ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, + if (tier_config.tier_placement.tier_type == "cloud-s3-glacier") { + ldpp_dout(dpp, 20) << "Restoring object:" << dest_obj << " from the cloud" << dendl; + RGWZoneGroupTierS3Glacier& glacier_params = tier_config.tier_placement.s3_glacier; + ret = rgw_cloud_tier_restore_object(tier_ctx, headers, + &set_mtime, etag, accounted_size, + attrs, days, glacier_params, &cb); + } else { + ldpp_dout(dpp, 20) << "Fetching object:" << dest_obj << "from the cloud" << dendl; + ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, &set_mtime, etag, accounted_size, attrs, &cb); + } if (ret < 0) { ldpp_dout(dpp, 20) << "Fetching from cloud failed, object:" << dest_obj << dendl; diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 5ecb2a1e17d8..0ea8140073ff 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -139,6 +139,7 @@ rgw_http_errors rgw_http_s3_errors({ { ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION, {404, "ServerSideEncryptionConfigurationNotFoundError"}}, { ERR_NO_SUCH_PUBLIC_ACCESS_BLOCK_CONFIGURATION, {404, "NoSuchPublicAccessBlockConfiguration"}}, { ERR_ACCOUNT_EXISTS, {409, "AccountAlreadyExists"}}, + { ERR_RESTORE_ALREADY_IN_PROGRESS, {409, "RestoreAlreadyInProgress"}}, { ECANCELED, {409, "ConcurrentModification"}}, }); diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index ab261d173fa2..44ac625dd5a1 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -355,6 +355,8 @@ inline constexpr const char* RGW_REST_STS_XMLNS = #define ERR_NO_SUCH_TAG_SET 2402 #define ERR_ACCOUNT_EXISTS 2403 +#define ERR_RESTORE_ALREADY_IN_PROGRESS 2500 + #ifndef UINT32_MAX #define UINT32_MAX (0xffffffffu) #endif diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 97b0be351524..4df987c970a6 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3614,8 +3614,9 @@ void RGWRestoreObj_ObjStore_S3::send_response() if (restore_ret == 0) { s->err.http_ret = 202; // OK } else if (restore_ret == 1) { - s->err.http_ret = 409; // Conflict - dump_header(s, "x-amz-restore", "on-going-request=\"true\""); + restore_ret = -ERR_RESTORE_ALREADY_IN_PROGRESS; + set_req_state_err(s, restore_ret); + dump_header(s, "x-amz-restore", "ongoing-request=\"true\""); } else if (restore_ret == 2) { rgw::sal::Attrs attrs; ceph::real_time expiration_date; diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 68442e741d98..45749c228f74 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1719,7 +1719,7 @@ public: /** Get the type of this tier */ virtual const std::string& get_tier_type() = 0; - /** Is the type of this tier cloud-s3/clous-s3-glacier */ + /** Is the type of this tier cloud-s3/cloud-s3-glacier */ virtual bool is_tier_type_s3() = 0; /** Get the storage class of this tier */ virtual const std::string& get_storage_class() = 0; -- 2.47.3