diff --git a/changelogs/unreleased/9015-Lyndon-Li b/changelogs/unreleased/9015-Lyndon-Li new file mode 100644 index 000000000..51021f8c0 --- /dev/null +++ b/changelogs/unreleased/9015-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8958, add VGDP MS PVB controller \ No newline at end of file diff --git a/config/crd/v1/bases/velero.io_podvolumebackups.yaml b/config/crd/v1/bases/velero.io_podvolumebackups.yaml index 0eadd8e59..f77c5df4a 100644 --- a/config/crd/v1/bases/velero.io_podvolumebackups.yaml +++ b/config/crd/v1/bases/velero.io_podvolumebackups.yaml @@ -15,38 +15,41 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - description: Pod Volume Backup status such as New/InProgress + - description: PodVolumeBackup status such as New/InProgress jsonPath: .status.phase name: Status type: string - - description: Time when this backup was started + - description: Time duration since this PodVolumeBackup was started jsonPath: .status.startTimestamp - name: Created + name: Started type: date - - description: Namespace of the pod containing the volume to be backed up - jsonPath: .spec.pod.namespace - name: Namespace - type: string - - description: Name of the pod containing the volume to be backed up - jsonPath: .spec.pod.name - name: Pod - type: string - - description: Name of the volume to be backed up - jsonPath: .spec.volume - name: Volume - type: string - - description: The type of the uploader to handle data transfer - jsonPath: .spec.uploaderType - name: Uploader Type - type: string + - description: Completed bytes + format: int64 + jsonPath: .status.progress.bytesDone + name: Bytes Done + type: integer + - description: Total bytes + format: int64 + jsonPath: .status.progress.totalBytes + name: Total Bytes + type: integer - description: Name of the Backup Storage Location where this backup should be stored jsonPath: .spec.backupStorageLocation name: Storage Location type: string - - jsonPath: .metadata.creationTimestamp + - description: Time duration since this PodVolumeBackup was created + jsonPath: .metadata.creationTimestamp name: Age type: date + - description: Name of the node where the PodVolumeBackup is processed + jsonPath: .status.node + name: Node + type: string + - description: The type of the uploader to handle data transfer + jsonPath: .spec.uploaderType + name: Uploader + type: string name: v1 schema: openAPIV3Schema: @@ -170,6 +173,13 @@ spec: status: description: PodVolumeBackupStatus is the current status of a PodVolumeBackup. properties: + acceptedTimestamp: + description: |- + AcceptedTimestamp records the time the pod volume backup is to be prepared. + The server's time is used for AcceptedTimestamp + format: date-time + nullable: true + type: string completionTimestamp: description: |- CompletionTimestamp records the time a backup was completed. @@ -190,7 +200,11 @@ spec: description: Phase is the current state of the PodVolumeBackup. enum: - New + - Accepted + - Prepared - InProgress + - Canceling + - Canceled - Completed - Failed type: string diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 0288fbcce..9f8a5b689 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -34,7 +34,7 @@ var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccYK\x8f\x1b\xb9\x11\xbe\xebW\x14v\x0f{ٖ\xec\x04\t\x02\xdd\xc6r\x02\x18\x19\xc7\x03k2\xb9.EVK\\\xb1\xc9\x0e\x1f\x92\x95\xc7\x7f\x0f\x8a\x0f\xa9\xd5\x0fK\xe3\x04\x9b\xe5eF|\x14\xeb\xf9U\x15\xbb\xaa\xaa\x19k\xe5\vZ'\x8d^\x02k%~\xf1\xa8闛\xef\xff\xe0\xe6\xd2,\x0eog{\xa9\xc5\x12V\xc1y\xd3|Fg\x82\xe5\xf8\x1ek\xa9\xa5\x97F\xcf\x1a\xf4L0ϖ3\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B[mQ\xcf\xf7a\x83\x9b \x95@\x1b\x89\x97\xab\x0fo\xe6o\x7f?\xff\xdd\f@\xb3\x06\x97\xb0a|\x1fZ\xe7\x8de[T\x86'\x92\xf3\x03*\xb4f.\xcd̵\xc8醭5\xa1]\xc2e!Qȷ'\xce\xdfEb\xebD\xec1\x13\x8b\xebJ:\xff\xe7\xe9=\x8f\xd2\xf9\xb8\xafU\xc125\xc5V\xdc\xe2v\xc6\xfa\xbf\\\xae\xae`\xe3TZ\x91z\x1b\x14\xb3\x13\xc7g\x00\x8e\x9b\x16\x97\x10O\xb7\x8c\xa3\x98\x01d\xd5Dj\x150!\xa2\xb2\x99z\xb2R{\xb4+\xa3B\xa3\xcfw\tt\xdc\xca\xd6Ge&Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6\xe0\xe1\xc0\xa4b\x1b\x85\x8b\xbfjV\xfe\x8f\xf4\x00~vF?1\xbf[\xc2<\x9d\x9a\xb7;\xe6\xcaj\xb2\xd1SgƟH\x00\xe7\xad\xd4\xdb1\x96\x1e\x99\xf3/LI\x119y\x96\r\x82t\xe0w\b\x8a9\x0f\x9e&\xe8W\xd2\x10\x90\x8a\x10\x8a\x86\xe0\xc8\\\xbe\a\xe0\x90\xa8D\x1d\x8ds\xaa\x06w]\xb1M\xac\xc0K\x8fJ\xe2\x9ff2\xf7\x1d\xb2ſ\xe7\xdc♤\xf3\xaci\xaf\xe8>lq\x8aؕ*\xdec͂\xf2]Q\xc9J\xaa\xeb\x97\xd7b\xb5\xc8\xe7\"\x9d\xba\xba\xf1\xfd\xd5\\\xbauc\x8cB\x96\xa8\xa4]\x87\xb7\xc9\v\xf9\x0e\x1b\xb6̛M\x8b\xfa\xe1\xe9\xc3\xcbo\xd7W\xd30\xe6H\xbd\xa0 ñ\x8emvh\x11^b\xfc%\xbb\xb9,ڙ&\x80\xd9\xfc\x8c\xdc_\x8c\xd8ZӢ\xf5\xb2\x04K\x1a\x1d,\xea\xcc\xf6x\xfaWu\xb5\x06@b\xa4S \b\x940\xf9U\x8e\x1f\x14Yr05\xf8\x9dt`\xb1\xb5\xe8P'\x98\xa2i\xa63\x83\xf3\x1e\xe95Z\"C\xb1\x1d\x94 ,;\xa0\xf5`\x91\x9b\xad\x96\xff8\xd3v\xe0Mvf\x8f\xceC\x8cP\xcd\x149k\xc0\x1f\x81iѣܰ\x13X\xa4;!\xe8\x0e\xbdx\xc0\xf5\xf9\xf8H\xd1 um\x96\xb0\xf3\xbeu\xcb\xc5b+}Ahn\x9a&h\xe9O\x8b\b\xb6r\x13\xbc\xb1n!\xf0\x80j\xe1\xe4\xb6b\x96\xef\xa4G\xee\x83\xc5\x05ke\x15\x05\xd1\tR\x1b\xf1\xbd͘\uebae\x1d\x84t\x1a\x11R_a\x1e\x82\xd7\xe42\x89T\x12\xf1b\x05\x9a\"\xd5}\xfe\xe3\xfa\x19\n'\xc9R\xc9(\x97\xad\x03\xbd\x14\xfb\x906\xa5\xaeѦs\xb55M\xa4\x89Z\xb4Fj\x1f\x7fp%Q{pa\xd3HOn\xf0\xf7\x80Γ\xe9\xfadW1\x8b\xc1\x06!\xb4\x11$\xfa\x1b>hX\xb1\x06Պ9\xfc\x85mEVq\x15\x19\xe1.kuss\x7fsRog\xa1\xe4\xd4\tӎ\xa2\xc1\xbaE~\x15w\x02\x9d\xb4\x14\x19\x9ey\x8c\xd1\xd5SP\x86\x8a\xe9\xa4\\\xc68H\xd0`\x9c\xa3s\x1f\x8d\xc0\xfeJ\x8f\xe5\x87\xf3\xc6+\x1e[\xb4\x8dt1\xbdBml?\xf3\xb03\x92wGA\xbc\xbe\xc1\x01P\x87f\xc8H\x05\x9f\x91\x89OZ\x9d&\x96\xfef\xa5\x1f^4aH\x1a\x89\xc5\xf5I\xf3'\xb4҈\x1b¿\xebm?\xab`g\x8ePG\xff\xd7^\x9d\b\xbb\xdcI\xf3!j\x97\xf1\xf0\xf4\xa1 x\x8a\xad\x1c\x98YWsx\xc8Amjx\x03B:*$\\$:T\x96\x0e*\x16\x1aK\xf06\xbcJ|nt-\xb7C\xa1\xbb\xb5є\xc7\xdc \xdd\xd3\xdc*\xdeD\xa8E\xde\xd1Zs\x90\x02mE\xf1!k\xc93'\xc1\xa6\fRKTb\x80M\x93Q\x16E\xb1((\xa8\x99\xbaa\xc3\xd5yc\xac\xa4\x99\xd4Ƀ/\x04\"\xd6\xd8&\xa7f\xedQ\v\xecg\x9bȍ\x89\x80\xe6P\xc0Q\xfa]BJ5\x16w\xf0\xd5أ\xb1\xc7\xd3\xd8t\x8f\xf7\xe7\x1d\xd2Δx\x11\x1cr\x8b>z\x1b*r\x1fr\xa59\xc0\xc7\xe0\"\xd6\xf6q\xa2\x8cX\xf0\x95\xd3{<\r\x15\r\xb7\x8c\x9bK\xa1\t\x96c\x11\xb5\x84ﾻ-\xd2 \xbb\x95A\xa5{\x11\xd4b\x8d\x16\xf5\xa0\x9a(\xe39\xe6(r\x1a\xf20\xack\xe4^\x1eP\x9dbN\"\xf0\xfc\x116\xc1\x83\b\x18\xad\xc6\xf8\xfeȬp\xc0M\xd32/7RI\x7f\x02\xe9&\xe83\xa5\xcc\x11E\xb686\xad?\xcd\xe1\x83v\x9ei\x8e\xee\\\a\x91ƒ+0\x9dv\xe5(\x8e\x05\x1d\xb3c\x18\x98\xc87\xc6y\xe0h\xc9\x1d\xd5\t\x8e\xd6\xe8픰#\xe9\x90z@\xab\xd1c̈\xc2pGɐc\xeb\xdd\xc2\x1c\xd0\x1e$\x1e\x17Gc\xf7Ro+b\xb0\xcaೈ\x9d\xdd\xe2\xfb\xf8\xe7[\xbc\xc0\xb4\t'\xeep\xdeu\x8c\xf5\x13\x95\xb7~\x87)E\xac\x93\x0f\x1a\vT@\x90k7\xd9w\x13\xb2\x8e\x85\xddX]\xde\x1d\xc5\xe4c\xf9c\x8f\xc3\xd4\xf1\x15P\x01\xf8R]t[5\xac\xad\xd2n\xe6M#\xf9\xac/m\xf2\xfb\xaf\xe3OiV\xa4\x16\x92Sq{\x8d\x1b\xa5\x89\x13W=͈\x1a\xfa]\xce\x14Z\x8e\xab)\x89\x9bk\x85\x1b\x1c\x7f\xea\uef74\xbe\t\xbas\xfew\xe8\xa9\xeet\xa0\x91\xea\x03f\x87z\x8e\x80ɍքT\xde\x00;\xa7\x81\x1f\\?\xff\xbd\x12=7\x81\xefqD\xf1\x03Q\xdeōE\xc7\xe9\x18\xf1\x12\x1c\xc6\xc4t\x8b\r\xb8\x1d\x11\x9c\xad\xd0\xde\xc3\xcb\xea\x816\x9eK\b\x06\xab\a\xd8\x04-\x14\x16\x8e\x8e;\xd4\xd4u\xc9\xfa4~\x17\x8d\xe7\xc7u\xd1j\xac\xber\xdfTt;.C\xcaoK\u061cF\xea\xa5;\x84l-\xd6\xf2\xcb\x1dB>ōE\xe1-\xf3;\x90\xdaI\x81\xc0Fԟ\n\xd9\tAϵѧ\x8c9\xdf`\x9e\xafaCb\xe75\xf0Pt|#~\x9e\xf2\xb6\xb3\x16\xca\xef\x9cݮ\xeb\xe4\xa98\x1e\x95\xe8p~\x94\xf9S\xaa>\xf9H\x19q\xc5\xcc\xcb\xf0\xc4W\xaa\xd8\xf244\x16\xccT3\x19kѵF\v\xea9\xef\xaba/,\xff\xef*\xd9q\xb3V\xd7(\xd7[+V\xb8\xab\x8d\x8b\xcf`\xafn\xe4\xd2\xe3`\xb7M2\x1bG\r\xf6\xa5\x97\xeb\xc9\xf8\x8b\xb4p\xa3%W\xa7\xaf\x93\x8eꗠce\x1b\xab\xaa\xf9l\xe4\xc4{l-R\x06\x13K\x92\xcdƃ\xda\x1c\xe9p\x87Z*ˌN\xf9\x9ez[\xa6E~U\xa0\xa5\x11\xcaG\xa9\x14\xd5\x00\x16\x1bCʢ\xb2\xdcR5\xc7b\xadu\xf8\xcd\xfc\xcd\xff\xafeT\xccy\xea\x00Q|ƃ\x1c>\xadݧ\xee\xc7\x01\x95\x82\x0e瘡\x1f?\x95׆\x85\xcd\xdb~\x82Z*\xaa\xff:\xd0qGu0\xf20\xfcn\xfd\xf8\x83\x8b=\x10j\xef\xe0H\x16t\x91%jzL~\xe1\t\xceS\x12\xb9i\xffn\x01\xae\r(\xa3\xb7h\xcbk\x0f\x15xɛ\x8c\x05\x81\x9er\x95\xde\x02\xdf1\xbd\xa5\xc8\x18\x83\xfc\xc8p\xe6\xbe\xcb'yϤ\x83H=\xe1\x1dw\x19\xf4Y\x8e\xb54\xaf1\xe6\xf43\xfc\x99\xffl\xd9\xcbkoO\xefSP[,\xd1_,\xa9\x9c\x14]\xf9\xcb\xd3\xfce|\xfb\xfb\xc0\xf0\xdd\xff[\xd5\xf3_}\xa9\x18|\xa1\xf8U(\xa7\xa1:\xf7f\xf1\xfc1\xedJ\xef\xb5\xf9\b\xb0\x8d\t~$\xf7w\x1c~4\xa6\xe3ǘ\xd7\xf0\x18?1\xdd*OhO\xb1\b\x0f\xd6\xc67\xdd\xf2\xd6\x18\x91b,+ݏ\xc0\x0f\xbd/aݵ\xe1w\xb2;\xe4\x1a\xcd҃ɔi;v\xcdJ\xee΄\xcd\xf9\xa5~\t\xff\xfc\xf7\xec?\x01\x00\x00\xff\xff\x03f\x86Y\xc0\x1d\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z_s\x1b\xb7\x11\x7fק\xd8Q\x1e\x92\xcc\xf8\xc8\xdam3\x1d\xbe\xd9r\xd3Q\x9b\xa8\x1aS\xf6K&\x0f\xcbÒ\x87\xe8\x0e@\x01\x1ce6\xcdw\xef,p \xef\x8e )҉}/6\xf1g\xf1\xc3\xfe߅\x8a\xa2\xb8B#?\x90uR\xab\x19\xa0\x91\xf4ѓ\xe2_n\xf2\xf877\x91z\xba~y\xf5(\x95\x98\xc1M\xeb\xbcnޑӭ-\xe9--\xa5\x92^juՐG\x81\x1egW\x00\xa8\x94\xf6\xc8Î\x7f\x02\x94Zy\xab\xeb\x9al\xb1\"5yl\x17\xb4he-\xc8\x06\xe2\xe9\xe8\xf5\x9f&/\xbf\x9b\xfc\xf5\n@aC30Z\xacu\xdd6\xb4\xc0\xf2\xb15n\xb2\xa6\x9a\xac\x9eH}\xe5\f\x95L{eukf\xb0\x9b\x88{\xbbs#\xe6{->\x042o\x02\x990SK\xe7\xff\x95\x9b\xfdA:\x1fV\x98\xba\xb5X\xef\x83\b\x93N\xaaU[\xa3ݛ\xbe\x02p\xa564\x83;\x86a\xb0$q\x05\xd0]1\xc0*\x00\x85\bL\xc3\xfa\xdeJ\xe5\xc9\xde0\x85Ĭ\x02\x04\xb9\xd2J\xe3\x03S\ued40\b\x10\"Bp\x1e}\xeb\xc0\xb5e\x05\xe8\xe0\x8e\x9e\xa6\xb7\xea\xde\xea\x95%\x17\xe1\x01\xfcⴺG_\xcd`\x12\x97OL\x85\x8e\xba\xd9\xc8\xdey\x98\xe8\x86\xfc\x86A;o\xa5Z\xe5`<Ȇ\xe0\xa9\"\x05\xbe\x92\x0e\xe2m\xe1\t\x1dñ>\xdc2\x7fp\x98\xe7\xed\xcecc\x06\bn,\xe1nk\x84 \xd0S\x0e\xc0\x96\x9f\xa0\x97\xe0+b\xce\a\xc5B\xa9\xa4Z\x85\xa1(\t\xf0\x1a\x16\x14 \x92\x80\xd6d\x90\x19*'F\x8b\x89JD\a\xb0\xeeF\xa3\xa7x\xc3\xeb\x7foT\x03@\xf7Z\\\x00\xe5\xacs\xe3\xe2\xc1\xa9\x1f\xfaC'\xf5\xa3\xa2\xb0&\x1dޚZ\xa3 \xcb\xc7W\xa8DM,Y\x04oQ\xb9%\xd9\x030Ҷ\x87\x8d\x19\x82y\x9f\xe8\xf5f\xceaFg;s\xaf-\xae\b~\xd0epP\xacҖ\x06:\xed*\xdd\xd6\x02\x16\xe9\x14\x00\xe7\xb5\xcd*8#\x8e\xbb:\xba\x89\xec\xc8Άg\x1eFߣ\x9d\xfc\xe9\xa4d\x1b\x91Z\xe5-\xe8\xf5\x8a\xf2\xd6\x13\xa7\xd7/\xa3\xbb*+jp֭Ԇ\xd4\xeb\xfb\xdb\x0f\x7f\x9e\x0f\x86\x01\x8cՆ\xac\x97\xc9}Ư\x17\x1cz\xa30d\xf5\xff\x8a\xc1\x1c\x00\x1f\x10w\x81\xe0(A.\xead\x1c#\xd1a\x8a\xe2\x91\x0e,\x19K\x8eT\x8c\x1b<\x8c\n\xf4\xe2\x17*\xfddDzN\x96\xc9$A\x95Z\xad\xc9z\xb0Tꕒ\xff\xdd\xd2v\xac{|h\x8d\x9e\x9c\x87\xe0j\x15ְƺ\xa5\x17\x80J\x8c(7\xb8\x01K|&\xb4\xaaG/lpc\x1c?jK \xd5RϠ\xf2\u07b8\xd9t\xba\x92>\x85\xccR7M\xab\xa4\xdfLC\xf4\x93\x8b\xd6k릂\xd6TO\x9d\\\x15h\xcbJz*}ki\x8aF\x16\xe1\"*\x84\xcdI#\xbe\xb2]\x90u\x83c\xf7\xb4&~!ҝ!\x1e\x8e} \x1d`G*^q'\x85\xe4\xbb\xde\xfd}\xfe\x00\tI\x94T\x14\xcan\xe9\x1e_\x92|\x98\x9bR-\xd9\a\xf0\xbe\xa5\xd5M\xa0IJ\x18-\x95\x0f?\xcaZ\x92\xf2\xe0\xdaE#=\xab\xc1\x7fZr\x9eE7&{\x13\xd2\n\xf6e\xada5\x17\xe3\x05\xb7\nn\xb0\xa1\xfa\x06\x1d}fY\xb1T\\\xc1Bx\x96\xb4\xfa\xc9\xd2xqdoo\"\xa5:\aD;\xca_\xe6\x86J\x16,\xf3\x96wʥ\xec<\xddR[\xc0\xf1\xf2!\x9f\xf2\x0e\x80\xbf\xac\x97\x1b/:\xa5t\xfc\xbd\xc9\x11J\x80U\xcfa'o\xdc9\xcfz\xe8<\xfb_r\xe1\xdb=\x96\x8cv\xd2k\xbba\xc2\xd1{\x8f\x15\xe2\xa0l\xf8+Q\x95T_r\xbd\x9b\xb0\x13\xa4\x12\xccv\xda*4\xbb\xa2H5\x00\xd5j\xa5\xd9\xc4\xc6Ҁ[\xcf\xcbX\xc9\x1d\xf9\xfc]U\xa00\xda\xc9\x17\x95\nvy \xf4\xf3\xbd\xf1\xa5\x17Zׄc^*-\xe8ĝﴠ\x9c\xb0x+\xf8\n}\xc2Ƌl\xab\xd4>o\xf9\xd3\xea,q\x18-N\xe0\xeaND\xb0\xb4$K\xaa\xa4\xe4\xfb\x8f\xe5c\x19d\xfdLi\x1f\xe3a\xfb\x80#\x812\x8b\xf8\xf5\xfdm\n\x86\x89\x89\x1d\xf6\xbdxw\x92?\xfc-%\xd5\"\xe4\x0e\xa7\xcf\xcej.\x7f\xb7\xcb\b\"D\x04\xaf\x01\xc1H\x8a\x19\xf76\x1a\x83T\xce\x13\x8an\x90\x9d\xa0\xa5n\xeeE\xf4\xf4\aA\xf2\xb7\x8b\xda,\x13@\x8e\xb2%\xc7+\xb4\x04\xb5|\xcc\xd8O\xfc\xaeC\xae\xb8\x83\xf9+[\xcfo\xd7\xf0Mt^\xd7\xfc\xf3:\xc2ئ-}\x03\xdb\xc1\x89Vf\xe5jE\xbb\xa4tOY8\xccr\x80\xfa\x16\xb4\xe5\xbb*\xdd#\x11\b\xb3\x9cb| \xb1\a\xef\xa7W?_\xc37C\x1e\x1c8J*A\x1f\xe1\x15{\x9f\xc0\x1b\xa3ŷ\x13x\bz\xb0Q\x1e?\xf2Ie\xa5\x1d)Ъ\xdeĂ`M\xe04W\x94T\xd7EL\x10\x05<\xe1\x06\xf4\xf2\xc09ID\xac\x9a\b\x06\xad?\x9a$v|8n4\xfbYS\xfa\x9eg/!\x8bz\x96\xf5~\xb1\f䙜\b\xe5\xc2'p\xa2_j]\xc0\x89\xc7vAV\x91\xa7\xc0\f\xa1K\xc7|(\xc9x7\xd5k\xb2kIO\xd3'm\x1f\xa5Z\x15\xac\x8cE\x94\xba\x9b\x86\n~\xfaU\xf8\xe7ҋ\x87Z\xffSo?\xe8M|~\x16\xf0\xe9nz\t\aRv\xff\xfc\xd8u\x90\x0f\xf3.\xe1\x1c\xd3d\x9b\x7f\xaadY\xa5Z\xaf\xe7m\x1b\x14\xd1\x1d\xa3\xda|!\xdba>\xb7\x96\x11m\x8a\xaeUY\xa0\x12\xfc\x7f'\x9d\xe7\xf1K\x18\xdb\xcaOr.\xefo\xdf~I\x8bj\xe5%\x9e\xe4@\r\x13\xbf\x8f\xc5\x0eUѠ)\xe2j\xf4\xba\x91\xe5h5\xe7\U00037085\xb4\x94dO\xa4\x7f\xef\x06\x8bS\x82\x9a\xa9\x06\xb6k\xce\xca?=\xae2\t_\xbf\x8b{,-<ʯӪ\xf0\x80+\ah\t\x10\x1a4\xac\x11\x8f\xb4)b\xc6aPr\xba\xc0\x19\xc1\xb6k\x05hL\xcd1=f\x11\x19\x8a]\xfe۱\a]\xb8\xdf!\x86dE\x99\xbats\xf2^\xaa/Ȝ\xf7# \xbf/\xa3\xb6=\xccR\xab\xa5\\\xb56T\xa0\xfb\x9cRm]㢦\x19x\xdb\x1e\xaa\xb9\x8e2\xf2\x81\x97\x1c\xbf\xff\xfb\xdeҤ\xe1'\x1a\xae\xf9[\rڰ\xfb\x97!\xd56\xfbP\nx\xd4Fbfܒ\xf3{\xd6\xcb\x13\xd7\xd7\xe7\xd8XT\xcaKJ\xee\xeeq$S\x95v\x8a\xde%\xf0\xa92\xed\xf7óB?\xc37pu\xcf\xe5\xc8\x10w\x91o\x97\x8c\xd6p\xcd<\x1a2Z\x8cF\x86np49\xe8\xd9\xf7\x91\xee\xf7\x90\xc2S\xcc\x19]\xa4\xf8\xc4\xd4\xf14\x06G\x9f\x1e\x9e8\xed\xbe\xb4\x8fTj.\xbf\x06\xfd\xec\x8b\xda,\xfbdB\xff\u05ca\xce0d\xc3~\xa0\xf7J\xd5\x1d\x9ck\x04\xf5\xc9ŝ!Gaj$B\x19\xc5U\xde\x12eM\x02\xd2S\xe4\x99T\x16\xb4\xe4\x18\x1d\x8d45\":x\x87\v\x98\x87\x8a\xc0\x85n\xea\xd7nK\xb3u$B3/Ä\xfd\x88\xbdԶA\x1f\x1f\x06\n&q\x99\xf7\xca\xdalC\xce\xe1\xea\x94\xd1\xfe\x18W\xc5\xfeL\xb7\x05p\xa1[\xbfm\xd0\f\"\xd2\u05eeS\xb4\xf3zD\xd9\xd6\xc7P\xc7\xd1WI\xa5\x97m]\x87=}\xef\xb0{\xa6\x0e\xa8\x16\x94\xcf\xeb\x8e4\x88\x8e\x01\xacНb\xd5=\xaf\xc9Y\xdd֥\x1d5;8\xe2\xbe\xef\xe8)3\xba\xf7nܟ\xbcI&\x93\x99\xfb>X\xc3Y\xf7\xef\x0e\xba\xc4ܷM\xcdJ\xd7\xc9µ\xc7\x1aT\xdb,\xc82s\x16\x1bOn\xe4\xf8Q\x89>'s\xd5\xdfn\x7f\x12j\xa4\xd4u0\xba^l09\xafAHgj\xdcl\xef\x12rn\xb6\xaf|cz\xa7\xe4\xc9\xd2\r\x1d\xca!\x8e\xb7\x16\x03\xa6\xb7Z\x1d\xa8R\x93\x91K\xe5\xbf\xfbˑ\xa4]*O\xabQ\x18\xe9晝o\xf8\x94?\xe6\x84#9\x90Sh\\\xa5\xfd\xed\xdb\x13\xaa1\xdf.L&\xb2\xcb\xe7\x83C\fo\x1eݢN\x152Pw\x0e\xe7,\xfb\x1d\xfe\x19\xc3%Z<\x1fP8\x11\xaf\xba\xbf\xaa\xc8E\x859\x19\xb4\xec\x13\u008b\xda\xcd\xf8}\xf8\x058\x19\x1a\xe0\x9c\xed\xc6\xf4\xb7\xacP\xad\xb2\xfd\x11\xadB\x02\xa7\xed\xfe\xf3&\x9c\f@\xc3\v}\xceؓU\xa7\xbd\xc1\x80\\\xf4hw\x8fI\xfd\x91v\xb1}g\x9d\xc1\xaf\xbf]\xfd?\x00\x00\xff\xff\f\x19\xe2z\x0f%\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4ZK\x93\x1b\xb7\x11\xbe\xef\xaf\xe8Z\x1flWi\xc8HI\\)ޤU\x9c\xda\xc4V\xb6ĕ..\x1f\xc0A\x93\x03\xef\f\x80\x00\x18R\x8c\xe3\xff\x9ej<\x86\xf3\x00\xc9]\xca\xf2\xe2\xb2K<\x1a\x8d\xaf\x1b\xfd\xc2\x14EqŴ\xf8\x88\xc6\n%\x17\xc0\xb4\xc0O\x0e%\xfd\xb2\xb3\x87\xbfٙP\xf3\xed˫\a!\xf9\x02nZ\xebT\xf3\x1e\xadjM\x89oq-\xa4pBɫ\x06\x1d\xe3̱\xc5\x15\x00\x93R9Fݖ~\x02\x94J:\xa3\xea\x1aM\xb1A9{hW\xb8jE\xcd\xd1x\xe2i\xeb\xed\x9ff/\xbf\x9b\xfd\xf5\n@\xb2\x06\x17\xa0\x15ߪ\xbamp\xc5ʇV\xdb\xd9\x16k4j&ԕ\xd5X\x12\xed\x8dQ\xad^\xc0a \xac\x8d\xfb\x06\x9e\xef\x14\xff\xe8ɼ\xf1d\xfcH-\xac\xfbWn\xf4\aa\x9d\x9f\xa1\xebְzʄ\x1f\xb4Bnښ\x99\xc9\xf0\x15\x80-\x95\xc6\x05\xbc#64+\x91_\x01\xc4#z\xb6\n`\x9c{\xd0X}g\x84thn\x88B\x02\xab\x00\x8e\xb64B;\x0fʈ?\xb0\x8e\xb9ւm\xcb\n\x98\x85w\xb8\x9b\xdf\xca;\xa36\x06m`\x0e\xe0\x17\xab\xe4\x1ds\xd5\x02fa\xfaLW\xccb\x1c\r\xe0.\xfd@\xecr{b\xd9:#\xe4&\xc7Ľh\x10xk\xbcP\xe9\xf4%\x82\xab\x84\x9dp\xb7c\x9684\xce\x1f;ϋ\x1f'\x8aֱF\x8f\x99\xea-\r\\q\xe60\xc7Ӎjt\x8d\x0e9\xac\xf6\x0e\xd3I\xd6\xca4\xcc-@H\xf7\xdd_\x8e\xc3\x11\xf1\x9a\xf9\xa5o\x95\x1cb\xf3\x86z\xa1\xd7\x1d8!Ym\xd0d\x01R\x8e՟È#\x02oz\xeb\x03'\x81n\xbf\xff,+\xa4x\xa0\xd6\xe0*\x84(\x95\xa5S\x86m\x10~Pe\x90\xe0\xaeB\x13%\xb8\x8ajU\xa9\xb6\xe6\xb0J'\x06\xb0N\x99\xac\x145\x96\xb3\xb0*\xd2MdG\xa2\x1c\xee\xf9%4\xad4Ȳ\x9a\x96\xac\xd1\xcc\xcf\x10J\xe6\xd5\xed\xf5\x06\x1f\xa5j}H\xa5\xe2\xd8\xe1\x87\x13\xb6\x84\x05mT\x89֞\xb8\x01Dc\xc0ȻC\xc7Y\x80*\xf4s\x12?\xad\xae\x15\xe3h\xc0)\xa8\x98\xe45\xd21\x188ä]G\x15\x99\n0-\xbb\xdf\xeb!+\x1f\xe2\xc01v¬\xed\xcb`\a\xcb\n\x1b\xb6\x88s\x95F\xf9\xfa\xee\xf6㟗\x83n D4\x1a'\x92]\x0e\xad\xe7uz\xbd0<\xee\xff\x8a\xc1\x18\x00m\x10V\x01'\xf7\x83\xd6\xc3\x10-,\xf2\xc8S\x80GX0\xa8\rZ\x94\xc1!Q7\x93\xa0V\xbf`\xe9f#\xd2K4D&݅R\xc9-\x1a\a\x06K\xb5\x91\xe2\xbf\x1dmKXӦ5sh\x9d\xbf\x8cF\xb2\x1a\xb6\xacn\xf1\x050\xc9G\x94\x1b\xb6\a\x83\xb4'\xb4\xb2G\xcf/\xb0c>~T\x06AȵZ@圶\x8b\xf9|#\\\xf2ťj\x9aV\n\xb7\x9f{\xb7*V\xadS\xc6\xce9n\xb1\x9e[\xb1)\x98)+\xe1\xb0t\xad\xc19Ӣ\xf0\a\x91\xde\x1f\xcf\x1a\xfe\x95\x89\xde\xdb\x0e\xb6\x9d\b:4\xefB\x9f \x1er\xaat\tX$\x15\x8ex\x90\x02u\x11t\xef\xff\xbe\xbc\x87\xc4I\x90T\x10\xcaa\xea\x04\x97$\x1fBS\xc85\xe9<\xad[\x1b\xd5x\x9a(\xb9VB:\xff\xa3\xac\x05J\a\xb6]5\u0091\x1a\xfc\xa7E\xebHtc\xb27>^\x81\x15\xdd%\xb2\x00|<\xe1V\xc2\rk\xb0\xbea\x16\xff`Y\x91TlABx\x94\xb4\xfaQ\xd8xr\x80\xb77\x90b\xa8#\xa2\x1dY\xb6\xa5ƒ\x04K\xd8\xd2J\xb1\x16љ\xac\x95\x016\x9e>\xc4)o\x00\xa8e\x1d\xc9x\xd29\xa5\xa3\xf6&G(1,{\x06<9\xbc\xe8\x9f\xea\xa1\x7f귃\x95\x8fk\fje\x85SfO\x84\x83\x83\x1c+\xc4Q\xd9P+\x99,\xb1\xbe\xe4x7~%\b\xc9\tv\xec\x14\x9aLQ\xa0\xea\x19Ur\xa3芍\xa5\x01\xb7\x8e\xa6\x91\x92[t\xf9\xb3\xcac\x0eMH8\x84\x98\xd0\x0f%LJ^)U#\x1bcI\xee\xee̙\xc9\x01\xe6\x84彭\xab\x98K\xbc\xd1$\xd3J9Ŗ\x9a\x92O\x12\x87V\xfc\f_qG\x06\x06\xd7h\xd0G#\xc1\xf6k\xe5=\x84cB&\x9b\x16\x12\x01p*\xc3\xd9*(\x11r\x18\xdf\r8y?\xe0\x84\xa3\xccr\xfc\xfa\xee69\xc3\x04b\xe4}\xe2\xef\xce\xe2Cm-\xb0\xe6>r8\xbfwVs\xa9ݮ\x03\x13\xde#8\x05\f\xb4\xc0\x12\a\xde\x18\x84\xb4\x0e\x19\x8f\x9dd\x04\rƱ\x17\xc1\xd2\x1fe\x92\xda\xc1k\x93L\x80\x91\xe7\x11\x1c\xfe\xb9\xfc\xf7\xbb\xf9?T8\a\xb0\x92B3\x9fDa\x83ҽ\xe8\x12)\x8eV\x18\xe4\x94\x16\xe1\xacaR\xacѺY\xa4\x86\xc6\xfe\xf4\xea\xe7<~\x00\xdf+\x03\xf8\x89Q:\xf2\x02D\xc0\xbcsfIm\x84\r\a\xef(\xc2N\xb8\xca3\xaa\x15\x8f\a\xdc\xf9#8\xf6@79\x1c\xa1E\xa8\xc5C\xe6\xfe\x84v\xed\xa3\xb9\x03\x9b\xbf\xd2\xed\xf9\xed\x1a\xbe\t\xc6\xeb\x9a~^\a6\xba\xb0\xa5\x7f\xc1\x0e\xec\x84[f\xc4f\x83\x87\xb8\x7f\xa2,\xe4f\xc9A}\v\xca\xd0Y\xa5\xea\x91\xf0\x84IN\xc1? \x9f\xb0\xf7ӫ\x9f\xaf\xe1\x9b!\x06G\xb6\x12\x92\xe3'xE\xd6\xc7c\xa3\x15\xffv\x06\xf7^\x0f\xf6ұO\xb4SY)\x8b\x12\x94\xac\xf7!\x00\xde\"X\xd5 찮\x8b\x10 rر=\xa8\xf5\x91}\x92\x88H5\x19hf\xdc\xc9 1\xe2p\xfa\xd2L\xa3\xa6\xd4\x1ew_|\x14\xf5\xa8\xdb\xfbl\x11\xc8#\x91\xf0\xe9\xc2g \xd1O\xbd.@\xe2\xa1]\xa1\x91\xe8Ѓ\xc1Ui\t\x87\x12\xb5\xb3s\xb5E\xb3\x15\xb8\x9b\xef\x94y\x10rS\x902\x16A\xeav\xee\xcbH\xf3\xaf\xfc\x9fK\x0f\xee\xeb?\x9f{zO\xe4\xf9 \xa0\xdd\xed\xfc\x12\x04Rt\xffx\xdfu\x14\x87e\f8\xc74\xe9\xce\xef*QV)\xd7\xebYۆ\xf1`\x8e\x99\xdc?\xd3\xdd!\x9c[C\x1c\xed\x8bX\x03-\x98\xe4\xf4\xbf\x15\xd6Q\xff%\xc0\xb6Ⳍˇ۷\xcfy\xa3Zq\x89%9\x92Ä\xf6\xa98pU4L\x17a6s\xaa\x11\xe5h6\xc5\U00037704\xb4\x16h΄\x7f\xef\a\x93S\x80\x9a\xc9\x06\xba9O\x8a?\x1d\xdbd\x02\xbe~y\xf8TXx\x12\xaf\xf3\xaap\xcf6\x16\x98A`\xd00M\x1a\xf1\x80\xfb\"D\x1c\x9a\t\n\x17(\"\xe8\n\x83\xc0\xb4\xaeɧ\x87(\"C1ƿ\x11\x1ef\xfd\xf9\x8e\x01\x92\x15e\xaaJ-\xd19!\x9f\x11\x9c\x0f#F~_\xa0\xba\x9a]\xa9\xe4Zlb\xb5s\x8a\x94l뚭j\\\x803\xed\xb1\x9c\xeb$\x90\xf74\xe5\xf4\xf9?\xf4\xa6&\r?S`̟jPv\x9c\x1e\x06e\xdbLY)\xe0Ai\xc12\xfd\x06\xad\x9b\xdc^\x1a\xb8\xbe~\xca\x1d\vJyI\xca\x1d\xd2\xe0\\V\x1a\x15=\x06\xf0)3u\xea\x90\xe5e\x85\xfe\x04\xdb@\xd9=\xa5#C\xbe\x8b|\xb9d4\xa7W]N]Z\xf1Q\xcf\xd0\f\x8e\x06\xc3\xf9\x1eUC\xf2\x05\xed'T\x91\xc2\xebU\xc448G\x97\u07b4(쾴\x8eD\x89\x9dvȻB\xff%\x12\x7f=&\xe2k\xbf\x86\xc7K!\x1a\xecR\xff\xa1\xad\v\xc9\xdd\nA\x1b\xd4,[\x15\x02_\xb9\xb7\xbe\x84\xf9\xb5\rĄ\x85\xd6\"\xf7\x15\xb4\xc9\xde\x13\n\xe9E\x893\x87\x05\xad\xbf\xcc^\xe4\vS\xe11\xad\xffRrQ\x95jJf\n!K\xa8\xf9'\x9c\xf4\x8a\x97C\xec@\xae\xc3+PC\xee\xb3PJ\x92\xd7L\xd4\xc8!=\x11?\x91\xca\n\xd7\x14\xe2\x04\x1b\x97\xea8\x91\xbd\xe3\xf9\xdfiIf@\x98\x06<_R\x98\rZ\xcb6\xe7lޏaV(o\xc5%\xc0V\xaauy%\xff\xda\xc6{\xfa\xb4\x12[\xb6r44\x11\xccU\xc9\"\xacۺ\xf6k\xfa\xc6\xf5\xf0\xf9\x80\xe7j\x85\xf9\xb0\xf8D}\xed\x14\x83\x15\xb3砺\xa399\xa3\xd5y\x84\x93V\vNx\xbfw\xb8\xcb\xf4&c\x90\x19\xba\x8b\x16&34\xf9\x0e\xa0?\x18\n\xc89\xe4\xd2X\x96f\xf7ʞ\x19\xfb\xde_\xbd'\x81\x1d\xf9\xbbĶt\x05\xe8J\xd5ɜ\xf8\xd7q\xd96+4$\t\xff\xfe>r\xd2L\xf2\xbe\xd8r\x99\xfaa}Ҡ@)V\x9bb\xdd\xdc\xdfo\xa7\x80\v\xabk\xb6\xef\xce\xe2\xf3#\xba\xcc\xf9G\x84ÍJfE\xe3\xb1x\xeft\x19\xb8\xfbV!\x9f\xfc\xe5>8\x18\xb6\xe9\xa7\x03\xa3\xf1\xee\x1b\x84/\xb3Éx\xd5J\xa6m\xa5\xdc\xed\xdb3\xaa\xb1\xec&\xa6\xfbxȽ\xbc\xf5\xf5\xefSqRT\x85\f\xab\a\xeb\xf6$c1\xfct\xe5\x12-^\x0e(\x9cq\x8e\xf1K\x9a\x9c\vZ\x92\x15 \x03\xe4_?oƟ9\xbc\xe8>\x9d`.V\x91ˊ\xc9M\xb6\x96\xa5\xa4\x0f\xb6\x95\x99>E\xc3Yo7<\xd0\x1f\xe9\xe8\xb2\xea4\xe9\xf4\x9c\xf3\x1e\xed\xf8\xf0\xd7\xefiWݛ\xf8\x02~\xfd\xed\xea\xff\x01\x00\x00\xff\xff\xd9H\xdbA\x14'\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z_\x93۶\x11\x7fקع<$\x991\xa5\xc6m3\x1d\xbd\xd9\xe7\xa6smr\xbd\xb1l\xbfd\xf2\xb0\"V\x12\"\x12@\x01P\xb2\x9a\xe6\xbbw\x16 )\xfe\x81\xa4\x93<\x8e\xf9b\v\x00\x97?\xfc\xf6/\x16\x97e\xd9\x04\x8d\xfc@\xd6I\xad\xe6\x80F\xd2GO\x8a\x7f\xb9\xe9\xf6on*\xf5l\xf7\xddd+\x95\x98\xc3}\xe5\xbc.ߒӕ\xcd\xe9\r\xad\xa4\x92^j5)ɣ@\x8f\xf3\t\x00*\xa5=\xf2\xb0\xe3\x9f\x00\xb9V\xde\xea\xa2 \x9b\xadIM\xb7Ւ\x96\x95,\x04\xd9 \xbc\xf9\xf4\xeeO\xd3ﾟ\xfeu\x02\xa0\xb0\xa49\x18-v\xba\xa8J\xb2伶\xe4\xa6;*\xc8\xea\xa9\xd4\x13g(g\xe1k\xab+3\x87\xe3D|\xb9\xfep\x04\xfd\xa4Ň \xe7m\x94\x13\xa6\n\xe9\xfc\xbf\x92\xd3?J\xe7\xc3\x12ST\x16\x8b\x04\x8e0\xeb\xa4ZW\x05\xda\xf1\xfc\x04\xc0\xe5\xda\xd0\x1c\x1e\x19\x8a\xc1\x9c\xc4\x04\xa0\xdeg\x80\x96\x01\n\x11\x98\xc3\xe2\xc9J\xe5\xc9\u07b3\x88\x86\xb1\f\x04\xb9\xdcJ\xe3\x033C\x88\xe0<\xfaʁ\xab\xf2\r\xa0\x83G\xda\xcf\x1eԓ\xd5kK.\xc2\x03\xf8\xd5i\xf5\x84~3\x87i\\>5\x1btT\xcfF\x8a\x17a\xa2\x1e\xf2\a\xc6켕j\x9dB\xf1N\x96\x04\xa2\xb2A\xb5\xbc\xff\x9c\xc0o\xa4\x1b\xc3ۣc\x88և\x8d\xa7\xc1\x84y\x16\xe9<\x96f\x88\xaa\xf3j\x84%\xd0S\nԽ.MA\x9e\x04,\x0f\x9e\x9a\xad\xac\xb4-\xd1\xcfA*\xff\xfd_N\xf3Q\x136\r\xaf\xbeѪO\xcek\x1e\x85\xcepD\xc2\xdaZ\x93M2\xa4=\x16\x9f\x02ij\x80ם\xf7#\x92(\xb7;~\x11\n\x9b\x1e\xe8\x15\xf8\r\xc1k̷\x95\x81\x85\xd7\x16\xd7\x04?\xea<\xaap\xbf!Ka\xc52\xae`\x0f\x06ɺ\xd36\xa9:C\xf94\xae\xad\x855\xb2\x06\xfa\xeb\x7f\xe8\xb3\xd8Wn\t\x93\xf6Մ\xa2iX!\xb5J\x1b٫5=\xcb\xc0\xbaD*-\xa8\xc3\xda\b\x97t`\xac\xceɹ3\x86\xcfBzH\x1e\x8f\x03\x17)\xdaPX\xd3\x00\xaaL\xa1Q\x90\x05\xafa\x83J\x14\x14u\xe8-*\xb7\xaa-c\xac\xc2\xe6\xb5w\aӇ\xf2\xbe\x91י\x19a\x8aKw\xdf\xc50\x98o\xa8\xc4y\xbdV\x1bR\xaf\x9e\x1e>\xfcy\xd1\x1b\x06\xa6Ő\xf5\xb2\x89\xcc\xf1\xe9$\x9e\xce(\xf4\xf7\xfc\xbf\xac7\a\xc0\x1f\x88o\x81\xe0\fD.pQ\xc7W\x125\xa6ȑt`\xc9Xr\xa4bN\xe2aT\xa0\x97\xbfR\xee\xa7\x03\xd1\v\xb2,\x06\xdcFW\x85\xe0ĵ#\xeb\xc1R\xae\xd7J\xfe\xb7\x95\xed\x98p\xfeh\x81\x9e\x9c\x0f\x8eh\x15\x16\xb0â\xa2\x17\x80J\f$\x97x\x00K\xfcM\xa8TG^x\xc1\rq\xfc\x14\xacI\xad\xf4\x1c6\xde\x1b7\x9f\xcd\xd6\xd27\xe98\xd7eY)\xe9\x0f\xb3\x90Y\xe5\xb2\xf2ں\x99\xa0\x1d\x153'\xd7\x19\xda|#=徲4C#\xb3\xb0\x11\x15R\xf2\xb4\x14_\xd9:\x81\xbb\xdegG\x8a\x8eOH\xa2W\xa8\x87\xb3*{\x02֢\xe2\x16\x8fZ\xe0!\xa6\xee\xed\xdf\x17\xef\xa0A\x125\x15\x95r\\:\xe2\xa5\xd1\x0f\xb3)Պ\r\x9f\xdf[Y]\x06\x99\xa4\x84\xd1R\xf9\xf0#/$)\x0f\xaeZ\x96ҳ\x19\xfc\xa7\"\xe7YuC\xb1\xf7\xa1d\x81%;\x14\xc7\x011\\\xf0\xa0\xe0\x1eK*\xee\xd1\xd1\x1f\xac+֊\xcbX\t\xcf\xd2V\xb7\x10\x1b.\x8e\xf4v&\x9a*\xea\x84j\x87\xf1ma(g\xcd2\xb9\xfc\xaa\\\xc9:\x93\xac\xb4\x05\x1c\xad\xef3\x95\x0e\x01\xfc$3\xcap\xd1%\xb3\xe3\xe7uJP\x83Xu\x02y\x9d\xef\\\x9d\xa8\x8a~\xa2\xea>\xa3\x1ci\xc9h'\xbd\xb6\x87c\xa6\x1c\x9a\xc4I\xed\xf0\x93\xa3ʩ\xb8e{\xf7\xe1M\x90J0\xefԚ4\a\xa3(5\x00\xd5j\xad\xd9\xc9F\xea\x80\a\xcf\xeb\xd8\xce\x1d\xf9\xf4f\xd5\xc9\xcc&\x15\x1ckL\xe8֒\xc3m/\xb5.\b\x87l\x1a-.l\xfaIׁ\xc3Ҋ,\x85\xfc\x1fì\xd1!\x18{\x94\xaa\t\x1f\xb1\xe4\x06\xaf\x13\xfbXr\xb89\xa5\x9a\xd3v\bgRR\x12𫧇&\xed4\x96UC\x1fe\x96.?I\xb3\xe0g%\xa9\x10!Q_\xfev\xd2B\xf8yXE\x10!\xf6z\r\bFRN\xbd\xbc\aR9O(\xeaA\x0e7\x96\xea\xb9\x171\xa6\x9e\x04\xc9\xcf1?\xb2J\x009\xc6K\x01\xff\\\xfc\xfbq\xf6\x0f\x1d\xf7\x01\x98s%\x14\xce*T\x92\xf2/\xda\xf3\x8a '-\t>}дD%W\xe4\xfc\xb4\x96F\xd6\xfd\xfc\xf2\x974\x7f\x00?h\v\xf4\x11\xb9\xe8\x7f\x012rަ\x8d\xc6j\xa4\x8b\x1bo%\xc2^\xfaM\x00j\xb4\xa87\xb8\x0f[\xf0\xb8e\x8f\x89[\xa8\b\n\xb9\xa54\xfb\x00w\xa1x:\xc2\xfc\x8dC\xca\xefw\xf0M\f\x12w\xfc\xf3.\xc2h\v\x84n\xd49\xc2\xf1\x1b\xf4\xe0\xad\\\xaf\xe9Xh\x8f\x8c\x85\x13\x1a\xa7\x82oA[ޫ\xd2\x1d\x11A0\xeb)\x06b\x12#x?\xbf\xfc\xe5\x0e\xbe\xe9sp\xe2SR\t\xfa\b/\xd9\xc7\x037F\x8bo\xa7\xf0.\xd8\xc1Ay\xfc\xc8_\xca7ڑ\x02\xad\x8aC\xac7w\x04N\x97\x04{*\x8a,\x96b\x02\xf6x\x00\xbd:\xf1\x9dFEl\x9a\b\x06\xad?[\x8e\xd5<\x9cw\x9aq}\xd2<\xcf\xf3\x97P\xaf<\xcb{\xbfX\xae\x7f&\x13\xa10\xff\x04&\xbaG\x9d\x1b\x98\xd8VK\xb2\x8a<\x052\x84\xce\x1d\xf3\x90\x93\xf1n\xa6wdw\x92\xf6\xb3\xbd\xb6[\xa9\xd6\x19\x1bc\x16\xb5\xeef\xa1e3\xfb*\xfcs\xeb\xc6C\x9f\xe5Sw\x1f\x84|9\n\xf8\xebnv\v\x03M\x1d\xfd\xfc\xdcu\x92\x87E]\xd9\re\xb2\xcf\xef72\xdf4\xa7\xaaN\xb4-Q\xc4p\x8c\xea\xf0\x85|\x87y\xae,#:du\xc31C%\xf8\xffN:\xcf\xe3\xb7\x10[\xc9O\n.\xef\x1f\xde|I\x8f\xaa\xe4-\x91\xe4\xc4i!>\x1f\xb3#\xaa\xacD\x93\xc5\xd5\xe8u)\xf3\xc1j\xae\x95\x1f\x04+i%\xc9^\xa8\xfe\xde\xf6\x167U{\xa2\xean\xd7\\Uv;\x85\xc6m\xb4\x7fxs\x01Ǣ]\xd8`8\xea\xb0.:\x1bY\xec\x12gk\xcdsx\x82o=\x9e\x8e\\}P\xfd\xd5\r2m\xe5Z*,\x8e\x110\x1c\xc5\x14\x96\x18~%t_\xa21R\xad\xaf\xc2\xda\xf4\x8b\x16\xe4\xf9\xf8\x9e(\x9c\xbb\xed\xecs\xe5\xf5Y\xbb\xbb\xecR\xef\a@\x00-\x01\xf2\x9eXC[:d\xb1\x8a3(\xb9\x04\xe3*\xab.U\x97\x04hL\xc1uR\xac\xccR\xbe\xdet\xbfr\xadVr]w\"\xc7L\xa9\xaa(pY\xd0\x1c\xbc\xadN\x1d\x82\x92\xee\xd3m\xbc]\xd0\xf8\xfb\xce\xd2F\xdd\x17Z\x7f\xe9]\xf5\x1a\x82\xe3͐\xaa\xca1\x94\f\xb6\xdaHL\x8c\xb3\xb1\x8f\x1c\x9d'\xee\xee\xae1\xa9\xe8I\x178\x88g\xd0\xd4\x01\xbevĺ\xac\xaf\x8f\xac\xd1\x1d\xd3\xd9\xf1Z\a\xe5\xa35\x9fQ\xfa\b\xb3t\xafb\xb0\xc6h1\x19\x92֍m\x83\xc9cd\x1aN\xf4\x9d~0\x1b)xV\x9b'4\x9e\xafi\xf4\xc4륚\xf7\x98V}s\xe9\xc4\x05\xfbͭ\x1e>\x13\x1aO\xa2\xed\xc9\xdf\xd2\ay5\x14\x12\x1a\xb4V\xd4N\"Kj\x9b\x06\xb5\x9d\xd8c\x1b#\x86lc\xc9`\xd2\" 4\xd9]h4~\xed\xa24\xe9\xa0r$Bl\x1d}|$\xa1\xb9\xf3\x11\xe8)\xe3\xf7o\v \xe9\xe6Q\xbc\xee\xea\xdej\xdc\xd4I\x1a\x8b\x19s\x88-mᾥ\xb9hKQv\x94\xd7\x12\x16ő\bGX>a\xafP\x16$\xa0\xbd̽\x9a\xf9\x04\xe8qq\xf39\xc9/\xc99\\_\nZ?\xc5U\xb1\x93U\xbf\x02\xb8ԕ?a\x95_\xbbڵ\xae\xca\xc9J\x8bKH\x1e\xb5\b0\xd4\xe9+\xac1\x9a\x84Z\xba\xd7ZWa\fM\xc2KM?^\x93\n5-\xe4\xf3\xb1\x06\xce\xe4\xb0G\xda'F\x1b\x0fNL=\xd5a!15\xba_\xefN\xc6\xcel\xaa\xa6i\xe6\x922\xdb\xcb\xeb\xc4\xdc\x0f\xc1]\xaeb\xbb\xc6wK@h\xfb\xba\x1b]41 \\:\xab\xaa\\\x92eU\x84k\xedF'm\x05\x8cJt5\x97:\x9c\xb7\x12\x9a4\x1cE\xd5\xfd\xa5\xba!\x1d\xbc\xdck\x10ҙ\x02\x0f\xedf\u0089\x88]:ݞ?\xfaU\x13\xab8\xf3\x9c\xa8\xdb\xcew~\xdb?\x02H\x9f\xf7R7\xf9\xfdg|'?\x98o/\xf7?\xcf\x17\xceԝ\xfd?\xb6\xb8\xc5@\x16=\t\x97\x92E\xfd\xc7\x1f\xd7\xc7\xf8\xfeg\xfe\xc8\xf0\x9edo4\x18\x90\x8b\x8e\xec\xfa\n\xa9;R-\xdb\xfb\xd59\xfc\xf6\xfb\xe4\xff\x01\x00\x00\xff\xff\xec\xe3\xc3\ac%\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc=[s\xdb8w\xef\xf9\x15\x98\xf4a\xdb\x19\xcbi\xa6\x97\xe9\xf8\xcd\xf5:\x8d\xfb}\xebx\xec4\xfb\f\x91G\">\x83\x00\x17\x00\xa5h\xdb\xfe\xf7\x0e\x0e.$%\x90\x84d˛-^2\xa6\x80\x03\xe0\xdc\xcf\xc1\x01\xb2X,\xdeц}\x03\xa5\x99\x14W\x846\f\xbe\x1b\x10\xf6/}\xf9\xfco\xfa\x92\xc9\x0f\x9b\x8f\uf799(\xaf\xc8M\xab\x8d\xac\x1fA\xcbV\x15\xf03\xac\x98`\x86I\xf1\xae\x06CKj\xe8\xd5;B\xa8\x10\xd2P\xfbY\xdb?\t)\xa40Jr\x0ej\xb1\x06q\xf9\xdc.a\xd92^\x82B\xe0a\xea\xcd?^~\xfc\xd7\xcb\x7fyG\x88\xa05\\\x11\x05\xdaH\x05\xfar\x03\x1c\x94\xbcd\xf2\x9dn\xa0\xb00\xd7J\xb6\xcd\x15\xe9~pc\xfc|n\xad\x8fn8~\xe1L\x9b\xbf\xf4\xbf\xfe\x95i\x83\xbf4\xbcU\x94w\x93\xe1G\xcdĺ\xe5T\xc5\xcf\xef\bхl\xe0\x8a\xdc\xdbi\x1aZ@\xf9\x8e\x10\xbft\x9cv\xe1W\xbd\xf9\xe8@\x14\x15\xd4ԭ\x87\x10ـ\xb8~\xb8\xfb\xf6OO\x83τ\x94\xa0\v\xc5\x1a\x83\b\xf8\x9fE\xfcN\xc2B\tӄ\x92o\xb8Q\xbb\x1aD<1\x155DA\xa3@\x830\x9a\x98\n\bm\x1a\xce\n\xc4;\x91\xab\x1e\xa40J\x93\x95\x92u\amI\x8b\xe7\xb6!F\x12J\fUk0\xe4/\xed\x12\x94\x00\x03\x9a\x14\xbc\xd5\x06\xd4e\x04\xd4(ـ2,`ٵ\x1e\xef\xf4\xbeNm\xcc6\x8b\v7\x8a\x94\x96\x89\xc0m\xc1\xe3\x13J\x8f>\"W\xc4TLw[\r\xdb#T\x10\xb9\xfc\x1b\x14\xe6r\x0f\xf4\x13(\v\x86\xe8J\xb6\xbc\xb4\xbc\xb7\x01e\x91Uȵ`\xbfG\xd8\xdan\xdcNʩ\x01m\b\x13\x06\x94\xa0\x9cl(o\xe1\x82PQ\xeeA\xae\xe9\x8e(\xb0s\x92V\xf4\xe0\xe1\x00\xbd\xbf\x8e_\x90xb%\xafHeL\xa3\xaf>|X3\x13$\xaa\x90u\xdd\nfv\x1fP8ز5R\xe9\x0f%l\x80\x7f\xd0l\xbd\xa0\xaa\xa8\x98\x81´\n>І-p#\x02\xa5\xea\xb2.\xff.\x12u0\xad\xd9Y\x1e\xd5F1\xb1\xee\xfd\x80\x02q\x04y\xac\xa88\xc6s\xa0\xdc\x16;*\xd8O\x16u\x8f\xb7O_\xfbLɴ'J\x8f7\xc7\xe8c\xb1\xc9\xc4\n\x94\x1b\x87\xacia\x82(\x1bɄ\xc1?\n\xce@\x18\xa2\xdbe͌e\x83\xdfZЖ\xdf\xe5>\xd8\x1b\xd4:d\t\xa4mJj\xa0\xdc\xefp'\xc8\r\xad\x81\xdfP\roL+K\x15\xbd\xb0DȢV_\x97\xeewv\xe8\xed\xfd\x104\xe2\bi\xbd\x16yj\xa0\x18H\x9a\x1d\xc6VA]\xac\xa4\x1a(\x19;d\x88\xa3\xb4\xf0\xdb洈U\x8b\xfb\xbf\xccq\x99m\xff\x1eG[~\xb3+k\x05\xfb\xad\x05T\xa6N\xfc\xe1P_\xa9\x9ej\x1f6\xcbF\xfb\xd4\x1dE\xb4m\xf0\xbd\xe0m\te\xd4\xeb\a\x1b\xcc\xd9\xc6\xed\x01\x144z\x94\t+D\xd6\xfaؽ\x88\xeeWT\xe0T\x01\x11\xd2$\xe01\xe1\xe0\x11&\x10\x03I\x9a`G\x03ubœ[&D\xb4\x9c\xd3%\x87+bT{\x88F7\x96*Ew#\xd8\n\x1e\xc0\x8b\x90\x15\x81xU\xc3Y\x81$\x8f\n\x05\xf1\xf5\xe7E\x15\xd3VQ\x86]>HΊ\xdd\f\xben\x93\x83\x82\xb4z\xd9\xf5;$K\xa8\xe8\x86I\x95\x12\x03\xa9\xb0kϞwjZZ-\xe9\x81\xec۸\xcc\r'\x91UI\xf9<\xc7\x10\x9fm\x9f\xce:\x90\x02\x1dʸ\x15Omo\xbb\x97@\xe0;\x14\xadI,\x93\x90\xb2E\xd3$\x15i\xa46\xe3t\x1fW]\xa4\xef\x1c\xa5~\x9c`\x9a\x83\x9d%Y\xdd5\xaf\x84\x03Q-\x0e\x06\nY\n\xb0ۨ-Q\xbb\xbeJ\xb6\xae\xef(RȒj(\x89\x14\xa33#\xbb\xb4\x1c\xb4\x9f\xabD\xce\xe8\xf4\xd0E\xb7\x7f\xf4x\b\xa7K\xe0D\x03\x87\xc2Hu\x88\xcc\x1c\x94\xba\x96\xa3XGP\x99ЦC\t\xe860\x01\x92XN\xdfV\xac\xa8\x9c\x87a\xd9\x13\xe1\x90R\x82\xb6\xda\x04]\xe6\xdd\xd8&\xc9\x1c\xf9\xfd$Sڣk3b\xb5\x0f/\xa5Q\xba\x96\xa1\x86\xbb\x96Dm\xa7{\x0ft\x8b\xffn\xe4\xe4\xb6\xff\x7f\"6\x18\x93\x13\x98vB\xfe\t\xba\x9f\xd9<=ʷ\x18ၾ$w+\x02ucv\x17\x84\x99\xf0uN\x12(\xe7\xbd9\xfeĴ9\x9e\xe93I\x93#\x13g\"L\x9c\xe2OH\x174\x19O\xdebd\xd3\xe4\xaf\xfdQ\x17\x84\xad\"\xd2\xcb\v\xb2b܀\xda\xc3\xfeI\xaa>P\xe65\x90\x91c\xf5\b\xe6\tLQ\xdd~\xb7.\x8e\xee\x92`\x99x\xd9\x1f\xec|\xe3\x10A\f\xcd\xf3\f\\\x82\xf12SPc\x1cN\xbe\"6\xbb/\xe8T_\xdf\xff|\x18+\xef\xb7\f\xce;\xd8Ȍйv\xbd\xb7\xa3\xfe\xfa|T\x10~A\x1f(\x06U.\xe7rA(y\x86\x9ds]\xa8 \x96>4tΘ^\x01&\x7f\x90Ϟa\x87`\xd2ٜÖ\xcb\r\xae=C\xc2\xf5O\xb5\x01\x0e\xed\x9a|X\xec\xf0d? \"0\x86\xcfe\x03\u05fc($r'閩KB\v\xb8?a\x9bY\xacҟ\xa3\x9f\xfaD\x0e\xf8I;ZZ\x89\xa9\x98\xcfij@\x99\xc9%\xa8k\xdf(ge\x9c\xc8\xc9ȝ\xb8 \xf7\xd2\xd8\x7f0@\xd3\xc8(?K\xd0\xf7\xd2\xe0\x97\xb3`\xd4-\xfc\x9c\xf8t3\xa0\xa0\t\xa7\xe5-\xc2\xfa9?g\xd3,\xb7E\xdc3M\ue10dW\x1cJ2\xa7\xc2\xf4\xae\x9b\xceMT\xb7\x1a\xd3uB\x8a\x05\xda\xcc\xe4L\x1e\xdfR\r\xd0\xfd\xe2I\xfd\x84_\xad\xb1p\xbf\xb8$3\xa7\x05\x94!\xb2\xc4\xec'5\xb0fE\xe6|5\xa85\x90ƪ\xf0<\x8e\xc8T\xac~7DZO\x9e\xf5\xee\xb7\xef\x8b\xe7\x98/XX\x93\xb3\xf0\x10\x8c\xac3p\xe0uw9\xbf\x9f\x85\x95ٌ^\x81\x13f\xbb\x8e$Gǻ\xe6 \xe5\x05\xe8@+\x8e.\xce,uiY\xe2\x11\x1a\xe5\x0fGX\x94#x\xe1X\xd5\xd0[\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I\xae\xf1\xa4\x8c\xc3\xe07\x9f\x87\xeb\x81ɘ\xb2\xb1SY\xfe\xd9Pnm\xbfU\xe0\x82\x00w\x9e\x80\\\x1d\xf8E\x17d[I\xed\xcc\xf6\x8a\x01\xc7\xf3\x8a\xf7ϰ{\x7fa\xa7\x9f\x9d\xb2\xafd\xde߉\xf7·8P\x18\xd1ᐂ\xef\xc8{\xfc\xed\xfdK\\\xa9LN\xcd\xec6`њ6y\x1c*\x92\xc9\xfa\xae\r8\xa6\x9f\x9b\xef\x92\xf2\xdeɞ\xdam\x16\x8b6R\x9b\xcf\xe9\xbc\xe1\xc8z\x1e\u0088\xa1g\x9cȱ\xcdF\f>\x8f\x16\xf5\xbdu\"W\x06\x94\xcf%:\x1b\x10\xe2\x8f\x17Ff\xa9S\x99\xfebc2\x90\xc6\xfc\xaeE\xf0\f7\xb9\x83\x9b\x9c%\x1e\xe3\xb0Z\xbc\x1c\xe9\xed\xdf~\xef\xe53\xad\xe4ڿ\xfb\x1bym\x87\xba\x90uM\xf7O5\xb3\x96z\xe3F\x06\x9e\xf6\x80\x1c\xf5պEyε\xc8\x1d\x0f\xe1\xf9喙\x8a\tB\x83\xda\x00\xe5\x19\x8a\x92F\xa6rةVQM\x96\x00\"\xa6\xe8\x7f\x04W\xa2f\xe2\x0e' \x1f\xcf\xe0zDt\x9d\xd3ٽ\x894\x89\x94\x8f\x1f\x9c\xc9jdI\xb6\x15(\x180\xc6a\xde\x1d=U!M/eq\x84C\xda\xc8\xf2'MVLi\xd3_\x82&\xadΥ\xf5\x91\xe4\xb3\xeb\xfe\xcaj\x90\xad9'\x82o\xbbi\x06g\xcd5\xfd\xce\xea\xb6&\xb4\x96\xad3\xe6\x86\xd5\xf1TףwK\x99\x89\xc7V\x98\xbf1Ғ\xa0\xe1`\x80,a\x95>\xefM\xb5B\n\xcdJP\xa1J\xc1\x91\x8dI+\x98+\xcax\x9b:%J\xb5c#`q\xab\xd4I\x01\xf0\x177\xb2\x97w\xac\xe4v\x88\xa0̽\xe3A\x1a\x10\xb6\"\xcc\x10\x10\x85\xc58(\xa7\x92q\n\x8f\fD\r\xcb\xd5sy\n\xdc6\x10m\x9d\x87\x80\x05\n$\x13\x93)\xb7~\xf7O\x94\xf1s\x90\xcdr\xde'\xa9\x1e\x81\x96\xa7\xe4h~\xed\r' t\xab\xf0\xf0\xdf\xe9\x8e-\xe3yk\xb6\x94#\x9c\xb6\xa2\xa8\x00\x95\x90\x18\xea\x06\a\x9e\tm\x80\xe6\xf2\x82\xf5\x8aZ!\x98X\xe7\xd1.;\x11\xda5\x87\ua954\x1c\xe8\xf8)d\xd7,\xae\xdf@\x13\xfd\xdaM\xf3BM\xd4\x11\xc1\x1d\x9b#\x1d\xb2)j\x95\x16\xa1\xc6@\xdd8\x91\x93D\xb5\xa2o]Π\x88\x8e\t\xc3\xfd*^3\xbef\x82e\xd0v@\xd7;\xc1L\xdfy\xb4 \xce\xea<\xda\t\xa2;pJ\x86\xedn\x00\xc0\nh\x88Cp\xed\x91k\x8ep$\x97@hYB\xe9r\x97\xd6\x15\xf1a\x89+|\x1b)nH\xee\xeexO0\x8b\xb2\xa1\r\x82N\xccê\r,Z\xf1,\xe4V,0\x18\xd7G\xeb\x90\x13\xb3T/\x9dޜ\xac\x8c\xe6\xf5K\xbe\x9a\x9e\xd3BC~\xcd\xe7\xa9\xe0?\x9dA\xcbd\xf3\xcdQ\t\x8f).\x98\xd3k\xae\x00{\xe4\xc7\xd9UL\xcd?1\xd8\x1fJ߸b\xe9\x17\x95\xc5ݥA\xf5\x9c\xc2m\x05\xa6\x02\x15J\xb3\x17X\x92^N\x9e\x90v\xc1K\xac\x93\xb3L\x15\\dW\xfe\xb9W9\x87\xd1M\xcb\xf9\x85\xe5m\xda\xf2d8l$\x8a\xd8!geՏ\xa5=\x86\x9c\xea\x8bl<\xf6+-\x86\xf5\x85\xb1\n\"\x14\x18\xca0\xb3\xa7qj\xbfXX\xda;\xdf\x1f\x96S`\xfe/,\xff\x0f/=̨\x94\xc8Gcn\x95fDb\x02V\x82\xc1zh\xec\xea+|?_\xe8\xfbc\xe1\xd4@\xfd\xa5\xf1\x123\xea\xc2f\xa05\x01g\xaf\xde\x04\xadA\xab\x9d+\x10\xed\x80\xcf\x19\xda\xf1ׅ\xbb\x05\x11\xc0\xa4\xf8\xf5k\x05A|}\xf5>\xd3\xe4\x9fI%\xdbDU\xdf\x04\xcaf\xaa;\xe67<(\xf4\xf0\a\n`\xe8\xe6\xe3\xe5\xf0\x17#}\xd9\af\xd1\x12\x800(\xea2\xb3L\x94l\xc3ʖ\xf2 \xb5\xdd\x1d\x02\xc7@\x1d\x9f%\xa0IE\x04\xe3\x8e\x01\xc3\xf8\x01Ñ/\x8d;\x969Z\xc5M\xfb\xa2y\xd5!'ׄ\fk>F\xac\xe1\xb1\xc7\x17\xafR\x05\xfb\x87\xd4z\x1c_\xe1\x91\x13I\xccTs\x9cPÑY,\xf6\xe2\xf3\x96\x9c*\x8dcb\xee\xb3Ud\xbc~\x1dF\x16~\xe6k.\x8e\xc1\xce\xd9\xeb+ް\xaa\xe2mj)2+(^\xaf\x142/\xfa<\xa9\x14`>`\x19\xaf\x82\x98\xad}xQ@sҖfk\x1a\x8e\xa9d\x98\xa5N\x9e\x98\xbdY\xad\u009bU(\xbcm]\xc2$\x17M\xfexL\xe5A\x8c\x93~\xa1M\xc3\xc4\xfa\x90)rYg\x92m\xe6Y\xe6~o!\x03\x9e\xe9\x873]t8\x12\xfa\xba\xeb҉H2\xa4-\x990\xf2\x92\\\x8b\x9d\x87\x9b\x80\xd3\v\x1f\x854\a\x17\xd9첶\x8c\xf3\xfem-\x04;\r\xcaߙԴv\xab\x1a\xf3\xf6\x93t\x95j\xe0\x94\x9f\x148~ك\xd1ώ\xbe\xa5\xe7_\xb7ܰ\x86\x83\xf5\xe86\xacL\xde!3\x15\xec\"\x92\xff&\xf1\x86\xd4r\x87\x90\xbe]\xf4\x9eQ\xec\x9eq\x926\xbf\xc9\x13\xb6\x97Q\xcc~\\\x11{\x06\xcdrE\xf1\r\x8b\xd5߰H\xfd\xad\x8b\xd3g8k\xe6\xe7\xe3\x8a\xd0O>\x81\tG\xfd\xf7\xb2\x84\a\xa9\xcc\\p\xf2\xb0\xdf?q\x92\xda\v\xd8$/\x89\b]\x13\xbb\xc4\x10Ç\x17\xa7m*}\xe8\x19\xdc\xe9_di\xd76w\xc6\xf2\xb8\xd7\xfd\xe0\xae\xf2\n\x14\b\xf7\xcc\xc7\x7f>}\xb9\x8f\xf0S>\xaf\xf7\x8c\xf7\x9e\x97p\x1eL\xe9\x91\xe3\x8f\xe6|1\x93\xc3\x16\xfa\x00\xaf|.B\x1b\xf6\x1f\xf8\xaa\xdb\v\xd2A\xd7\x0fw\b#\xf8i\xf8L\\\xac\xa2\x88'\x96K\xb0\x16+\xa2jT,\xeeV\x03\x88Ê\xdf\xfe3JP\xba'\xb3\x82\xc5d\xa1\xc6\xcb\n\xdeÝ[\xc7\xd8,\x9f\xac\xd3(vD:\x8e\xac\x98*\x17\rUf\x87l\xa3/\x06k\bff*\x9d3\xaaX\x0f\x9f\x01K\xa27\xbc\xfe\x85g\x91\xbbfxڻ\x8f\xbbS\xd61~\xffd\xf6\xe6\xc9+\xaec\xdcb/\x10S\x89\xcf\xc9\x02\x93WK\x93yM\xf4\xf0\xed\xa4\xb4\xcbc\x1c=\xad\xe7l\x14\x1dRM\t0v<\xaa:-h\xa3\xabēK/\xd3u\xf8\x1a\x99\xa1\xa6}\xc9&\x1d\x80\xc1>YQ\xf5\xb4\xd5\x16\x82>\v\xdbFi\xc5a)\xddnm\xb3\xab{a\xfc\xa2\x97)x\x9b#\xe1\xcc\xe7\\N~\xc8šgD\xfd`\xf6˪\xb6CL\x9dp\x18<\xeb\xdae\x14\x19O;\xb1\x99π\xe4\x19\x8c\x13\x9e\xfe@|\xe5\xe2\x8a$_\x04\xc9|\xf5\xe3\x0fE\xf4\x84V\xd3E\x05e\xcb\xe1\xd47\xff\x9ez\xe3\xe7_\xfd\v\xb3e\xbc\xfbg\x91\xdd3\xd0\xd6g\x1e\xbe/\xe8)\xe1!\xf7)9\xe6\xf0ap\xe0\x9e\x17+\xdcK\x94E\x01Z\xafZ\x1e\xaa\x94\n\x05\xd4@\x19\xba3\x1dW|T\x9dM\xdbpIKP7R\xacX\xe2\x84d\x80\xd6\xff\x1at\xde\xe3\xd9\x02?\xb6\xaa{\xdaq\xf2Y\xbc\x17i\xae\x86*\xca9\xf0O\x8c\x83\xfeYn\x85]W\x86@>\xa4\xc6\xf5\xeee\x15\xad\xb2f}GD[/\xad\x93\vƌ\a\x8b+\xa9\xa6+\xa4\x1dޙ0\xb0\x86T|\xbdU\xcc\xc0SC\x95\x06\\Q\xc6\x0e~\xdd\x1b\xe2\xa2\xcf\x15\xa7kW\nW\xb2\x82\x1a\x88\x06\x18g\x18[>\x8e\xd7\b\x8b\xef\xb02I\x8e$\xbd\xb2\x85z\xecJƨX\x8f=/\x9a0\xd5\xc9\aF\x9dE.hc\xf0\x02\f\xd2\x11\x89h<\f|\xb4w\xef\x8d\xd1\x01\xd8qN\xf3e̾`N\x1bZ'\xa2\x84y\xbdss\b\x06\x9f\x05Ve\xaf\xee\xae\xff\xc0b,\xb0#[\xaac1u\xd2\xf7\xee`;0\xe8\xaa[\xd0P\x12\u0600 V\x14)\xe3PNq\xeaWL$\xab\r\xa8\x9ft\x84\x83\x95\x80\x96ş\fU&.\xfdЏYIUSsEJj`aG\x9f溥\x9fIU\xea\xc4\xe3@\xbc\xd9\xe6ţ\b\xd7n\xac\xf5s\xf7\xd1jК\xaeC\x10\xba\x05\x05d\r\xc2\xe2=\xe6\x16\x93\x1eS\xb8\xd2\xe7\x8dE,-\xb5(\xa4\x85i\xa9\x9f\xc0\xb9p\xf1\xf44\xbcO\x8cQ\xeczTE\xa7U\x85\xbf<\xf8\bT\xef?w}\x80\x8bO\xfd\xbe>I\xecv\xec\xceF\xa8+\xf0\xc4\a\x8f\r\x8b\x91uJ\xa6\x8dę\x8f2'\x95\x94\xcfYn\xf6\xe7رK'1\xe1X\t\xafL.ekz~\x8eGxb\x99\xf8\xfc\xe7+\xdb\x17\x84y\xed.P\x8d\xe5V\xf3<\xbd\xcf\x03H1\xbc\x95\x86\xf2`d,_\xc6\x0e\xd5\xc4\x03\x02O\xe1\xf1d\xcew\x17\xfb\x90\xf7^e\xef`W\xddS\x9e^\x13t\xd7\xc7\xc7Ҫ>\xeb\x97\x04\x12_\x01\xed|\x92\xb17\x17\xe7\xec\x1fB\xfd\x84\x8b\xca\xc0\xf1\xe7\xae\xf7\x18\x1e\xdd2\x9d\xc3\f\"\x1di\x12\f>L\x15%ㄥOx\xa9ME\xf5\x9c{\xfa`\xfbD\xb7\xa3g\xae\xa2\x13\xfa8\"\x95\xe9{\xae\vr\x0f\xdb\xc4W\x87,<\xfdB\xa9Jt\xb9\x13\x0fJ\xae\x15\xe8C\xa6[\xe0}F&֟\xa4z\xe0횉/\xe3\x95\xdfS\x9d\x1f\xa82\xcc2\xad[Ob\xecM\xb0q\x89\xdf\xe6G\x8f\xff\xc0\x04\xe5\xec\xf7\x94.\xef\xff87Ä\xbek<\xf2N\xb1P\x01\xf1s\n\xd0k\xe8\x9ft\xcf\xfc\x84y/ɽL\x8a\xb1? fC\xa0L\x93%h\xb3\x80\xd5J*\xe3\xf2\xf7\x8b\x05a\xab\xe0 Y\r\x81q\xa2{͞\xb0T\xe2=\x1e\xbd\x05\x87e\xe5S\x89\n\xad\x0e\x86\x9c5ݹ\x8c$-\n\x1b\x13\xc0\amh*6y\x91\x9e\xc6P\xd5\xcbJ\x8e\n\xb9\xeb\xf7\x8f9\xbe\xa8>\x10\x9cC\x1d^gw\x06\x9d\x8f\x9di\r^\xcb \xdab\xef\x14eB\x9c\x1a\xbb\x1b\x0f\xbb\xf3L\xcd\xd7\beL=\xfa\xfd\r\x1e\xe2\xf6\a\xac\xbe\x93%[QQ\xb1\x1e\xbd\xd0V)ٮ\xab\xc0\x9bc\x0e\x11)[\x8c\x9c\x1bT\x05:\xfc\xc7!\xa6U\xa2wh\xe7k,ƴt\\\uee0f\xf2\x02E\xad\xba\x8b-\x9d\xaa\x9a\xb0\xf9\xd9Y\xc2\x11\x88\xb3\xb6?\x01\x91\xea\x9d(&\xaf\xe0\xf8@\x9bM\xdc՝\xc2P\x12\tQ\x1b\xbf\x1a\x12\"\xc41$\xf4}\x89.\xe2\xf9a02棜\x88\x8ei'\x06\xb78\rj~\xd3}'h\xe8\xee\x1c\x87\x0e=\b\xfeNJ\xbb\r \x1c\x13\xf9\xe2\xdc\xe9\xb8\xf7ǍX7\xd1ۺ=9v\xfd\xb6\ac\xef\n\xa4\x8db\xbbiB\xbc\xf9\xf7l\x95\x92\x17\xf7\xbf3-9\xfc\xc3\xc1\xafo|\x95qK\x95`b}\x12F~\xf5c\x13\xf1\xbc\a{Έ>\xac\xfc\xd5b\xfa\xa4Y:\xf8\x88\f^\xf6\xf0\xecg\xf2_\xfe/\x00\x00\xff\xffP\a\xb5\x16Cm\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfa\x15\x84\xeea?B\xdd^\xc7}ą\xde|\xb2gO\xb1\x1e[ai\xf4\xbctU\xb6\x9aQ\x15\xd4\x00\xd5r\xdf\xde\xfe\xf7\x8dL\xa0\xbe\xba\xe8\xa2Z-ygǼت\x86$\xc9L\xf2\x03\x12X,\x16g\xbc\x12\xf7\xa0\x8dP\xf2\x92\xf1J\xc0W\v\x12\xff2\xcb\xc7\xff6K\xa1\xdelߞ=\n\x99_\xb2\xab\xdaXU~\x01\xa3j\x9d\xc1{X\v)\xacP\xf2\xac\x04\xcbsn\xf9\xe5\x19c\\Je9~6\xf8'c\x99\x92V\xab\xa2\x00\xbdx\x00\xb9|\xacW\xb0\xaaE\x91\x83&\xe0\xa1\xebퟖo\xffk\xf9\x9fg\x8cI^\xc2%3\xd9\x06\xf2\xba\x00\xb3\xdcB\x01Z-\x85:3\x15d\b\xf4A\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xeb\xdbӧB\x18\xfb\x97\xde\xe7\x8f\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5\b\xf9P\x17\\\xb7\xdf\xcf\x183\x99\xaa\xe0\x92}®*\x9eA~Ƙǟ\xba^0\x9e\xe7D\x11^\xdch!-\xe8+U\xd4e\xa0Ă\xe5`2-*K#\xbe\xb5\xdcֆ\xa95\xb3\x1b\xe8\xf6\x83\xe5g\xa3\xe4\r\xb7\x9bK\xb64ToYm\xb8\t\xbf:\x129\x00\xfe\x93\xdd!n\xc6j!\x1f\xc6z{Ǯ\xb4\x92\f\xbeV\x1a\f\xa2\xccrb\xa0|`O\x1b\x90\xcc*\xa6kI\xa8\xfc\x0f\xcf\x1e\xebj\x04\x91\n\xb2\xe5\x00O\x8fI\xff\xe3\x14.w\x1b`\x057\x96YQ\x02\xe3\xbeC\xf6\xc4\r\xe1\xb0V\x9aٍ0\xd34A =l\x1d:\x1f\x87\x9f\x1dB9\xb7\xe0\xd1\xe9\x80\n»\xcc4\x90\xdcމ\x12\x8c\xe5e\x1f\xe6\xbb\aH\x00F$\xaaxmH8\xda\xd67\xddO\x0e\xc0J\xa9\x02\xb8\x80vX4\xb6\nu%\xa0\x80\xe6\f\xddN\x8d\x16FH\xb6\xae\xd1#]2\xd4\x12Q\x19\x11\xd2X\xe0\x11a>\x01\xef\xe0kV\xd49\xe4WEm,\xe8\xdbLU\x90\x87E\xa6Q͜\xca\xc3\x0f\a!\xfb\xf8\xa5\x10\x19 \x1f2WiA\x8b<1\xd1nC\x99]\x05n\xcd\tY\xed\x87\xd0\xc6(\x93\xbaŀņ\xe7\x7f<\xbf \t\xe8\xf7\xde\xef\xc70\xae\xa1!\xd3,\xddL\x16\x7f\xbc\x85\xb0PF\xa8;\xa9\xa3f\xf0\x9dk\xcdw\a\xb8\xde,\xa6\xbd\x00\xdfc\xb0\a\x9c\x97\xa1\xda7\xe2\xfd\xb0\xff\xdf\"\xf7O\xcboC\x8b\xce\\H\xe4s!\x8c\xed\xb1ٸU,$\xebX\b\xe9\t$\x1dLT\x93S\\\xfd'!\xe6I\xe7Nl\xb24\xb2\xe9'\xc0\xbf\x14%7J=\xa6P\xef\x7f\xb1^\xbb\x84\xc52\xda\x18a+\xd8\xf0\xadP\xda\f\x97I\xe1+d\xb5\x8dj\x16nY.\xd6k\xd0\b\x8b\x96\xf9\x9b]\x81C\xc4:\x1c\xbe\xb0\x8eʊV\x18\x8c\xabe:\xb2\x94\xa8\x11\x1b\n\x05\xa8Q\xa8\xce\xc1\xc1Ђ\x1c\x88\\lE^\xf3\x82|\t.37>\xde\xe0\x17\xd3j\x13\x02\xb1\x87\x7fT\xaa]q\x0eM\x18$2\xb1\xb7\xea\xa5$\xa0\x8f_bl\xb4_5N\x89\xb0\x94p\xb0od\xa6\xae\v0\xbe\xbb\x9c\xdc\xe4V']\xb4\xccrk\f\x05_A\xc1\f\x14\x90Y\xa5\xe3\x14J\x91\x03WR\x95n\x84\xb8#Z\xb6\x1fm\xb5\x83\x99\x00\xcb(\xc4݈l\xe3\xdcW\x144\x82\xc5r\x05\x86VExU\x15\x11\xd3ՖI\xe1\xf0\x9dM鍶$h\x90!ܘ.iK\xa2~n\xcb(\xd9۹٧\xfa\xf8:\xff(\xbe\xbf%\xa2\a\xabs\xa4\xb0Oh\x12F\xfb\x05\xc9\xf3!Jz\xa4\xb8\x00\xb3\xec\xac\xce\t\x1b\xbe\xa60\xb4\xe7?\xeem\xa5\xec\x11\xe5\xd7Ż\xe3&\xcc\f\xd6MΩ\x97e\\\xd3Ϳ\b\xdf\xc8d\xddz\x8b5\x8bg\x1f\xbb-/hW\xc03$\xbf`kQX \xa7j\nQ6\x83s\xa7$P\xaa\x05f\xb4Il\xb3͇f\xef(\xa1ŀVC\x00\xceA\x0fQ\x0e\xf1 \x01$k\\\v\xda4\x15\x1aJڌ\xa5H\xb2\xfb\x85\\\xc1w\x9f\xde\xc7c\xcfnI\x94ԽA%LZW\xde\r\x1c\xa3.\xae>T\t\xbf\x90\xbf\xd6\x04\x82n\x13\xfe\x82q\xf6\b;\xe7bqɐo)\x8b\xff|\xf8*\fv,s\xf6^\x81\xf9\xa4,}yQ*\xbbA\xbc\x06\x8d]O4A\xa5\xb3$H\xc4n\U00088ce5(\xa8\r?\x84a\xd7\x12C2G\xa2\x19\xddQ\xae\x90\xeb\xd2uVֆ\xb6Z\xa5\x92\v\xb7,6֛\xe7\x81\xd2=\x16\x9c\xa4c\xdf\xe9\x1d\x1a#\xf7\x8b\xcbZ*x\x06yآ\xa3t\x1an\xe1Ad3\xfa,A?\x00\xab\xd0,\xa4K\xcb\fE\xedG6_\xbc\xd2=\x87n\xf9\xbax\xacW\xa0%X0\v4k\v\x0fŪ2\x91.\xde&\x8c䜌\x95\x05\xce\xf5ĚAZ\x92\xaaG2r\x0eWO%\xd63\xc9D^\x04\xb9]IR\xd0Ml\x9dg\xbdf\xca\xcd1*\xa63\x16\xe7\x02\x94\x9c\xb6\xd6\xfe\x86\x96\x9ef\xe3\xdfYŅ6K\xf6\x8e2{\v\xe8\xfd\xe6\x17&;`\x12\xbb\xadh\x95\xfd\x97Zly\x81\xfe\a\x1a\bɠpވZ\xef\xf9j\x17\xeci\xa3\x8cs\x1b\x9aM\xbb\xf3Gع\x1d\xe5\xa4n\xbb\n\xeb\xfcZ\x9e;_fO\xf14\x8e\x8f\x92Ŏ\x9d\xd3o\xe7\xcfu\xeffH\xf4\x8c\xaa=Q.y\x95.ɔ7;'\xd0\xc0`=8DظI \xc5\x00a\x8a\x02ɢ\\)\x13I\x16\x89\xa0\x95 \xe87\xcaX\xb7\x0e\xd9\xf3\xf7G\x17*UX\x9cd|mA3c\x95\x0e)\x99\xa8\xf8S\x96\xe2\xbb\xe5n\x03\x06\xfc>\x94_\xf4t\x801\x8a=ou\x83\xb3*\xe7n/\x8c:\xe2\x19yOԶ\xd2*\x03\x13͋hK\xa2m\xeaQp\x9f\x0eͺ.w\xd1\xdf:Ik\xa7,J\x872ϑG\xd2\x1d\x11\x19}\xf8\xdaY\xa2F\xed\x82\x7f\xa7H\xeb182:\xafQ\x96|\x98\x0e\x9c\x8c\xee\x95k\x1d\xe6\x98\a\xe6\xc2-\xfdP\x93Ι\xe3u4\xa2\xfc\xcf\xe6ڔB^SG\xec\xed\v\xbaC^\x8b\xc7ң\xc6\xca\xf1N\xfaU\xe8\xac\xe5^\xf3\xc1\xe7\xd4)\xda\xf8\xd1\xd0c\xee\xfe\x9e\by\xd7R\xd9\xce2\xceL'\xbaR\xf9\xef\f[\vml\x17\rs \xb1j\x14\xd4\x11\xa1\xa7\xfc\xa0\xf5ё\xe7g\u05fa\xb3\xa0\xb8QO>qzN\xbc\x1dH\xba\xe1[\xf0\x99\xab 3UKZ\nC=\x80\xdd̀\xe8X\xe3\xac@\xa2\xbd\xeb4\x96u\x99N\x90\x05I\x92\x90\x93\xebf\xdd&?p\x91\xb6nŎc\xab=\x94\xc39V\x8e\x9fG!\xc1\xb3\x9bN_\U000af8acK\xc6K\xe4!\xb9\x1d\xa2\x84&\xa3ޱ\xbbI\xfb\xc4\x16d\xb4\xac\xc2YV\x15`\xc1\xa7m\xce\xc0#S҈\x1c\x1a\xd3\xefE@I\xc6ٚ\x8b\xa2\xd63\xb4\xeal\x92\xcf\r¼69}d\x95\x8eȂH\x94\xb8\xce>\xc3\v\x9e\xd6\xf8\x95\x9e\xe7Ǧ8\x8c\x1a\xe6\xfb\x8b\x95\x16\xca\x1d\x068\xbd\xcb\xe8ӎ\xb9\xdc}\xf7\x19\xbf\xfb\x8c\xdf}\xc69\x1d}\xf7\x19'\xcaw\x9f\xf1\xbb\xcfx\xb8|\xf7\x19S\xcaw\x9fq&\"\xdf\xcagL\xc1pAk\x9c\a*$a\x95\x98\n1\x85\xf6D_>\xe9ǟ\xd58I.\xf3\xf58ȑC<\x91\xe3\x171\xaf\xa35^Mr3\xce\xc00w\xdc)\xca\x04\x87\xf9\x04\xa7g\x02\x02\xa7?=s}\x10\xf2\tO\xcf\xf8!\xa4E\x18G\x9d\x9d\tD\x9a\x7fz\xe2\xc2'\x11\x95\xc0\xc3V\x8aK\xff\x88\x8d1&I\tx|\xe3\xe4\xf7\xbd\x8c\xc9\x17\x90\xa5W9\x913K\x9eFY\x7f\xfe\xc7\xf3_\a\x8bN˔(\x1b\xf6i\xeb\xd4xL?b,\xdfM\x8d\xecg\xa9\xfez\xa6\xc2Ie?\xf5DMC\xe4\b\xbc\xbeX\x0f\xa8\xfck\xd27\x16\xcaϕ\xb7\x96'8a\x7f=\x02/\xe9\x8c=7;\x99m\xb4\x92\xaa6~M\ba\xbd\xcbܽ\x03\x01dL\xd8G5\xc8\x7f\xb0\x8d\xaa#\xa76&H\x9b\x90E\x9bF\x90^R\xadO\x8c\x00˷o\x97\xfd_\xac\xf2)\xb6\xecI\xd8M\x04\x18\xddG\xc1\xf3\x1c\xe3\x82\u0381\x1e\xaf\a\xc2UIC\xa1\x8c\x00S\x9aIQ8\x89\r\x10z\xf2\xca>Wnu\xf0h\xbfiz\r+=\x11wn\xfam\x93-9\xed\xbe?#\xe9\xf6\xa4G\xa3\xbeYZ\xedqɴ\xa9+\x94\t\x89\xb3\xe9\xe9\xb2)lu%=I69BNM\x88\x9d\xbb\x02\xf1\xa2ɯ/\x93\xf2\x9aL\xb3\xb4\xf4ֹ\x14{\x95T\xd6WN`}\xbd\xb4\xd5\x19ɪ\xa7?\xf5\x92\xbe\x96~tveڲ\xcc\xe1\x84Ӥ4Ӥ\xa5\x9b\x94\x01\x1f5Ԥ\xf4ѹI\xa3I\x9cL\x9f\xae\xaf\x9a\x16\xfa\xaaɠ\xaf\x9f\x02:)m\x93\x15\xe6&y\x8e_r\x18ʴ\x03P|\v\xe1|.\x99\x94\xee\xb9\xe6ϊ;?\x0f`\xa1\xb0\x047\xf5\x15〲.\xac\xa8\x8a\xf6>\xb6X\xc0\xb9\x81]sY\xd1ϊ\x8e\xc8\xfb\x9b\xba>\x7fi$~9\x88j\xb8aOP\x14\x8c\xc7\xe6\xe6\x1e\x152w\x0fh\xa6\x16\x80\xb6\x11g\xb9\xbf\x8c\xc9_\x1ez\xe1\xa6\v\xdd\x06@\x16\xb6\x8c-\xf5qy\xf8\xa6\xaf\x83\x06,U\x8f\xedy\xe6.ޠo\xbfԠw\x8c\xee\x1dk|\xb3\xf6P\xa9\x9f\xe8\x06\x03Ӡ~\xbc:<\xb4g\xb2\x17\xe0\xb4ꁽ\x93\xce#\x18\xe2DmP\xef\xb4\x01\x1d*U\x8cӢ\xfdD@H\xd5@\x884Mq\xfe眲|\x89\xf0\xee\x14\x01^\x92\a4\xcf{\xfd\x86\xa7'\x8f=5\x99\x9e\x8c\x92tJ\xf2%½9\x01\xdf,\x7f5\xfd\x14\xe4\xfc\x8d\xe7\x17>\xf5\xf8R\xa7\x1dgP/\xf5t\xe3|ڽ\xd2i\xc6W?\xc5\xf8\x9a\xa7\x17g\x9dZLNϚ\x95q0'\xb5\xea\x19\xc7\xed\xd2r\t\xa6O!&\x9e>L\xcc4H\x1b\xfc\x91\xc3N<]8\xffTa\"\x7f\xe7L\xe9W>=\xf8ʧ\x06\xbf\xc5i\xc1\x04\tL\xa82\xffT\u0cf7\xa4\x94\xceAOn\xfb͑\xdaIyM\x8d\xe5\xfa\x88\r\xf6\xb5\xc2m\xb2X\xab\x17\x03\x90Y\xf2\x17\xf9ӣ\r\x87\xb6\xc1Q2;\x1eQo_\xb2u\xd7\xfa\x0e\xb1\x7f\xcd\xc1m]\x1a\xa88\x1a\x00\n\xdc(5+\xea*|\xe0\xd9f\xd0Æ\x1b\xb6V\xba䖝7\x9b\xc5o\\\a\xf8\xf7\xf9\x92\xb1\x1fT\x93\xabӽ/͈\xb2*v\x18\x89\xb1\xf3n\x83\xe7IIT:C\xcf7\xaa\x10Y\xc4\xe7\x1c\xbdW\xcf5ػl\x88n\xfe\xcb:\xd9\"\xb1\xc0\a\x9b\x8bp\xebb\xffJfw\x9f\xfb\x91k%\xbc\x12\x7f\xa6'\x95N\xb0\xea\xf6\xee\xe6\x9a`\x051\xa2\xb7\x9a\x9a\x04ņ\xe5+@\x97\xa1\x1d\xfb!}r\xbd\xeeA\xed\xe7\bw\x1f\xab\x80ܽL\x12\xdc\x16\xaf\x9a3\x85Z\xeb\xe6\xda\xe1r\xa8'\x94/.wL\xf9\xa7'\x84\xce\x17\x15\xd7v璉.zx\x04\xbb>\xb5jv\xd0Z\xed\xbf\xbc\xd2-=\xb2\x87GWh'{W\xf5\x93\a\x86\xf4|\x0eN\x87OUO\x9e\xa7~\x01\x9c\x0e\xbbP\v\xa2b\xe4\xa7h\x06\xe4\xc9W,\x8d\xbf\xa1\xffG\xb5\x85\xf7ѕ\xcb\xfe\xeb+\x83&#\xa9\x89\x01*]2\x1f\xa1`\x9b\x8fHw|?O\xed\xc5s\r\x03*\xfe\x8e\xf0\xe7,N\xde\xf6A\x8d?HB7\xa8\x87Nc^\x15=\xf5\xb4c7\xf7\x14\xb76\xaa\xd4O}\x1f\xb7\x86\xe5ɐ`\x10\x81%\xe4\xc17ZNEF\xab4\x7f\x80\x8fʽ\xad\x93\"&\xfd\x16\xbd\x97\x97\xbc\xe7\x16\xf2\xb5\xfd$\x8c)z?\xb6!\xc0\xf6|\xc6\xdeE\xff\x88\xed\x91O\x19X[\xba\x91ғ&\xef\xfd\xeb$\xa8\x8f\r \v\x02\x05\x1c\xb4\x15\xfew\xa3\x9e\xe8\x02\xfc\xf8\x1asx@\xa4\xf3\x86\x19\xd0A\x11J\xe1=j\x98uU(\x9e\x83\xbe\xa2GT\x12F\xfcS\xaf\xc1\xc0\x1d\xe8?\xc5\xe2\xedfd<\xa1\xe7\x17̒A\x8f\xae(\xa0\xf8A\x14`\x1c≦\xe1f\xbfec)\xear\xe5<\xd55\xfe\xd8tr\xc02\xbb\xa1\xd2\x06C\x05\x1a\xfdD\xb7\x15Q\x9b \xf9\x87\x89\xc1\x1a>\ni\xe1\x01\xc6c\xe8\t\x9b\xe0\xdeh \a (0\x8a\xf8\xfe\x12[y\xec\x11\xe4>\xdez \x03\xcdbdL\x8e\x95w\xabn\xee\xaf\f\xabeN\x1b\x00\xf7\x7f\xbe=J~\xb7\xbd\xf7e\x82NHQ\xef\xf7\xe3-;!BG;\x91O\x1fW\xe21X\xdc\x18\x95\t\x8a*\x9e\x84\xf5\xd79\xbe\xdc\x1d\xe2\x87\x02\xc4\x03\xd2Q\x1b\xf8\xfc$A\x7f\t\x16\xc8\\\xcbػ-\xd3\xda\xef\xa7=h\xd1\xf7Z\xac¾G`\f\x000\x15\xf6\xb9\x8c{\t(l\xaf\t\xd3H\x1c\xc4>\x01\xdcyG\xc8ik\x85\xb4\xe3\x9c!n\x9bVt\xd8tDCN\x8b\xed\xfd\x00\xc6 \x93\x9d\x1e}j\xaa\xb8Ӧ\x86\xfd^\x8cy\xa3\xb4c\x96\xe1@\xff\xb0\xf7kT\x83\x1f\xd4\xde1\xcd=\xaaF\xf6>\xd2CxyGr\xbc\x97\xde\xfdR\xaf\xda\a\x15\xd8\xdf\xfe~\xf6\x8f\x00\x00\x00\xff\xff)\x00\x87w>{\x00\x00"), diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 102475edb..ad24b97ba 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -105,6 +105,9 @@ const ( // defaultVGSLabelKey is the default label key used to group PVCs under a VolumeGroupSnapshot DefaultVGSLabelKey = "velero.io/volume-group" + // PVBLabel is the label key used to identify the pvb for pvb pod + PVBLabel = "velero.io/pod-volume-backup" + // PVRLabel is the label key used to identify the pvb for pvr pod PVRLabel = "velero.io/pod-volume-restore" ) diff --git a/pkg/apis/velero/v1/pod_volume_backup_types.go b/pkg/apis/velero/v1/pod_volume_backup_types.go index ced436a82..546616c5a 100644 --- a/pkg/apis/velero/v1/pod_volume_backup_types.go +++ b/pkg/apis/velero/v1/pod_volume_backup_types.go @@ -64,12 +64,16 @@ type PodVolumeBackupSpec struct { } // PodVolumeBackupPhase represents the lifecycle phase of a PodVolumeBackup. -// +kubebuilder:validation:Enum=New;InProgress;Completed;Failed +// +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed type PodVolumeBackupPhase string const ( PodVolumeBackupPhaseNew PodVolumeBackupPhase = "New" + PodVolumeBackupPhaseAccepted PodVolumeBackupPhase = "Accepted" + PodVolumeBackupPhasePrepared PodVolumeBackupPhase = "Prepared" PodVolumeBackupPhaseInProgress PodVolumeBackupPhase = "InProgress" + PodVolumeBackupPhaseCanceling PodVolumeBackupPhase = "Canceling" + PodVolumeBackupPhaseCanceled PodVolumeBackupPhase = "Canceled" PodVolumeBackupPhaseCompleted PodVolumeBackupPhase = "Completed" PodVolumeBackupPhaseFailed PodVolumeBackupPhase = "Failed" ) @@ -113,20 +117,27 @@ type PodVolumeBackupStatus struct { // about the backup operation. // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` + + // AcceptedTimestamp records the time the pod volume backup is to be prepared. + // The server's time is used for AcceptedTimestamp + // +optional + // +nullable + AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"` } // TODO(2.0) After converting all resources to use the runttime-controller client, // the genclient and k8s:deepcopy markers will no longer be needed and should be removed. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="Pod Volume Backup status such as New/InProgress" -// +kubebuilder:printcolumn:name="Created",type="date",JSONPath=".status.startTimestamp",description="Time when this backup was started" -// +kubebuilder:printcolumn:name="Namespace",type="string",JSONPath=".spec.pod.namespace",description="Namespace of the pod containing the volume to be backed up" -// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.pod.name",description="Name of the pod containing the volume to be backed up" -// +kubebuilder:printcolumn:name="Volume",type="string",JSONPath=".spec.volume",description="Name of the volume to be backed up" -// +kubebuilder:printcolumn:name="Uploader Type",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer" +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="PodVolumeBackup status such as New/InProgress" +// +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this PodVolumeBackup was started" +// +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" +// +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where this backup should be stored" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this PodVolumeBackup was created" +// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the PodVolumeBackup is processed" +// +kubebuilder:printcolumn:name="Uploader",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer" // +kubebuilder:object:root=true // +kubebuilder:object:generate=true diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 47cc8b199..c998a1656 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -1043,6 +1043,10 @@ func (in *PodVolumeBackupStatus) DeepCopyInto(out *PodVolumeBackupStatus) { *out = (*in).DeepCopy() } out.Progress = in.Progress + if in.AcceptedTimestamp != nil { + in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeBackupStatus. diff --git a/pkg/builder/pod_volume_backup_builder.go b/pkg/builder/pod_volume_backup_builder.go index ac89f21de..e16ed223f 100644 --- a/pkg/builder/pod_volume_backup_builder.go +++ b/pkg/builder/pod_volume_backup_builder.go @@ -119,3 +119,33 @@ func (b *PodVolumeBackupBuilder) Annotations(annotations map[string]string) *Pod b.object.Annotations = annotations return b } + +// Cancel sets the PodVolumeBackup's Cancel. +func (b *PodVolumeBackupBuilder) Cancel(cancel bool) *PodVolumeBackupBuilder { + b.object.Spec.Cancel = cancel + return b +} + +// AcceptedTimestamp sets the PodVolumeBackup's AcceptedTimestamp. +func (b *PodVolumeBackupBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *PodVolumeBackupBuilder { + b.object.Status.AcceptedTimestamp = acceptedTimestamp + return b +} + +// Finalizers sets the PodVolumeBackup's Finalizers. +func (b *PodVolumeBackupBuilder) Finalizers(finalizers []string) *PodVolumeBackupBuilder { + b.object.Finalizers = finalizers + return b +} + +// Message sets the PodVolumeBackup's Message. +func (b *PodVolumeBackupBuilder) Message(msg string) *PodVolumeBackupBuilder { + b.object.Status.Message = msg + return b +} + +// OwnerReference sets the PodVolumeBackup's OwnerReference. +func (b *PodVolumeBackupBuilder) OwnerReference(ref metav1.OwnerReference) *PodVolumeBackupBuilder { + b.object.OwnerReferences = append(b.object.OwnerReferences, ref) + return b +} diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 286ca7439..c7e987ddc 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -48,7 +48,6 @@ import ( snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned" - "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/buildinfo" @@ -60,7 +59,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/nodeagent" - "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" @@ -282,30 +280,6 @@ func (s *nodeAgentServer) run() { s.logger.Info("Starting controllers") - credentialFileStore, err := credentials.NewNamespacedFileStore( - s.mgr.GetClient(), - s.namespace, - credentials.DefaultStoreDirectory(), - filesystem.NewFileSystem(), - ) - if err != nil { - s.logger.Fatalf("Failed to create credentials file store: %v", err) - } - - credSecretStore, err := credentials.NewNamespacedSecretStore(s.mgr.GetClient(), s.namespace) - if err != nil { - s.logger.Fatalf("Failed to create secret file store: %v", err) - } - - credentialGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore} - repoEnsurer := repository.NewEnsurer(s.mgr.GetClient(), s.logger, s.config.resourceTimeout) - pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), s.kubeClient, s.dataPathMgr, repoEnsurer, - credentialGetter, s.nodeName, s.mgr.GetScheme(), s.metrics, s.logger) - - if err := pvbReconciler.SetupWithManager(s.mgr); err != nil { - s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerPodVolumeBackup) - } - var loadAffinity *kube.LoadAffinity if s.dataPathConfigs != nil && len(s.dataPathConfigs.LoadAffinity) > 0 { loadAffinity = s.dataPathConfigs.LoadAffinity[0] @@ -328,7 +302,12 @@ func (s *nodeAgentServer) run() { } } - if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, podResources, s.logger).SetupWithManager(s.mgr); err != nil { + pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, podResources, s.metrics, s.logger) + if err := pvbReconciler.SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerPodVolumeBackup) + } + + if err := controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, podResources, s.logger).SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller") } @@ -347,7 +326,7 @@ func (s *nodeAgentServer) run() { s.logger, s.metrics, ) - if err = dataUploadReconciler.SetupWithManager(s.mgr); err != nil { + if err := dataUploadReconciler.SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data upload controller") } @@ -358,7 +337,7 @@ func (s *nodeAgentServer) run() { } dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, restorePVCConfig, podResources, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics) - if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil { + if err := dataDownloadReconciler.SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data download controller") } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index 11c0e8da4..c22819e58 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -312,30 +312,29 @@ func (f *fakeSnapshotExposer) DiagnoseExpose(context.Context, corev1api.ObjectRe func (f *fakeSnapshotExposer) CleanUp(context.Context, corev1api.ObjectReference, string, string) { } -type fakeDataUploadFSBR struct { - du *velerov2alpha1api.DataUpload +type fakeFSBR struct { kubeClient kbclient.Client clock clock.WithTickerAndDelayedExecution initErr error startErr error } -func (f *fakeDataUploadFSBR) Init(ctx context.Context, param any) error { +func (f *fakeFSBR) Init(ctx context.Context, param any) error { return f.initErr } -func (f *fakeDataUploadFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[string]string, param any) error { +func (f *fakeFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[string]string, param any) error { return f.startErr } -func (f *fakeDataUploadFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error { +func (f *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error { return nil } -func (b *fakeDataUploadFSBR) Cancel() { +func (b *fakeFSBR) Cancel() { } -func (b *fakeDataUploadFSBR) Close(ctx context.Context) { +func (b *fakeFSBR) Close(ctx context.Context) { } func TestReconcile(t *testing.T) { @@ -651,8 +650,7 @@ func TestReconcile(t *testing.T) { } datapath.MicroServiceBRWatcherCreator = func(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { - return &fakeDataUploadFSBR{ - du: test.du, + return &fakeFSBR{ kubeClient: r.client, clock: r.Clock, initErr: test.fsBRInitErr, diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 16ad49c06..6fde05784 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -27,23 +27,29 @@ import ( corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/vmware-tanzu/velero/internal/credentials" veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" - "github.com/vmware-tanzu/velero/pkg/podvolume" - "github.com/vmware-tanzu/velero/pkg/repository" + "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/uploader" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" + "github.com/vmware-tanzu/velero/pkg/util" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( @@ -52,36 +58,41 @@ const ( ) // NewPodVolumeBackupReconciler creates the PodVolumeBackupReconciler instance -func NewPodVolumeBackupReconciler(client client.Client, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, ensurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter, - nodeName string, scheme *runtime.Scheme, metrics *metrics.ServerMetrics, logger logrus.FieldLogger) *PodVolumeBackupReconciler { +func NewPodVolumeBackupReconciler(client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, + nodeName string, preparingTimeout time.Duration, resourceTimeout time.Duration, podResources corev1api.ResourceRequirements, + metrics *metrics.ServerMetrics, logger logrus.FieldLogger) *PodVolumeBackupReconciler { return &PodVolumeBackupReconciler{ - Client: client, - kubeClient: kubeClient, - logger: logger.WithField("controller", "PodVolumeBackup"), - repositoryEnsurer: ensurer, - credentialGetter: credentialGetter, - nodeName: nodeName, - fileSystem: filesystem.NewFileSystem(), - clock: &clocks.RealClock{}, - scheme: scheme, - metrics: metrics, - dataPathMgr: dataPathMgr, + client: client, + mgr: mgr, + kubeClient: kubeClient, + logger: logger.WithField("controller", "PodVolumeBackup"), + nodeName: nodeName, + clock: &clocks.RealClock{}, + metrics: metrics, + podResources: podResources, + dataPathMgr: dataPathMgr, + preparingTimeout: preparingTimeout, + resourceTimeout: resourceTimeout, + exposer: exposer.NewPodVolumeExposer(kubeClient, logger), + cancelledPVB: make(map[string]time.Time), } } // PodVolumeBackupReconciler reconciles a PodVolumeBackup object type PodVolumeBackupReconciler struct { - client.Client - kubeClient kubernetes.Interface - scheme *runtime.Scheme - clock clocks.WithTickerAndDelayedExecution - metrics *metrics.ServerMetrics - credentialGetter *credentials.CredentialGetter - repositoryEnsurer *repository.Ensurer - nodeName string - fileSystem filesystem.Interface - logger logrus.FieldLogger - dataPathMgr *datapath.Manager + client client.Client + mgr manager.Manager + kubeClient kubernetes.Interface + clock clocks.WithTickerAndDelayedExecution + exposer exposer.PodVolumeExposer + metrics *metrics.ServerMetrics + nodeName string + logger logrus.FieldLogger + podResources corev1api.ResourceRequirements + dataPathMgr *datapath.Manager + preparingTimeout time.Duration + resourceTimeout time.Duration + cancelledPVB map[string]time.Time } // +kubebuilder:rbac:groups=velero.io,resources=podvolumebackups,verbs=get;list;watch;create;update;patch;delete @@ -93,13 +104,13 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ "podvolumebackup": req.NamespacedName, }) - var pvb velerov1api.PodVolumeBackup - if err := r.Client.Get(ctx, req.NamespacedName, &pvb); err != nil { + var pvb = &velerov1api.PodVolumeBackup{} + if err := r.client.Get(ctx, req.NamespacedName, pvb); err != nil { if apierrors.IsNotFound(err) { - log.Debug("Unable to find PodVolumeBackup") + log.Warn("Unable to find PVB, skip") return ctrl.Result{}, nil } - return ctrl.Result{}, errors.Wrap(err, "getting PodVolumeBackup") + return ctrl.Result{}, errors.Wrap(err, "getting PVB") } if len(pvb.OwnerReferences) == 1 { log = log.WithField( @@ -108,157 +119,420 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ ) } - // Only process items for this node. - if pvb.Spec.Node != r.nodeName { + if !isPVBInFinalState(pvb) { + if !controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { + if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { + if controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { + return false + } + + controllerutil.AddFinalizer(pvb, PodVolumeFinalizer) + + return true + }); err != nil { + log.WithError(err).Errorf("Failed to add finalizer for PVB %s/%s", pvb.Namespace, pvb.Name) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + + if !pvb.DeletionTimestamp.IsZero() { + if !pvb.Spec.Cancel { + log.Warnf("Cancel PVB under phase %s because it is being deleted", pvb.Status.Phase) + + if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { + if pvb.Spec.Cancel { + return false + } + + pvb.Spec.Cancel = true + pvb.Status.Message = "Cancel PVB because it is being deleted" + + return true + }); err != nil { + log.WithError(err).Errorf("Failed to set cancel flag for PVB %s/%s", pvb.Namespace, pvb.Name) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + } + } else { + delete(r.cancelledPVB, pvb.Name) + + if controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { + if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { + if !controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { + return false + } + + controllerutil.RemoveFinalizer(pvb, PodVolumeFinalizer) + + return true + }); err != nil { + log.WithError(err).Error("error to remove finalizer") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + } + + if pvb.Spec.Cancel { + if spotted, found := r.cancelledPVB[pvb.Name]; !found { + r.cancelledPVB[pvb.Name] = r.clock.Now() + } else { + delay := cancelDelayOthers + if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { + delay = cancelDelayInProgress + } + + if time.Since(spotted) > delay { + log.Infof("PVB %s is canceled in Phase %s but not handled in reasonable time", pvb.GetName(), pvb.Status.Phase) + if r.tryCancelPodVolumeBackup(ctx, pvb, "") { + delete(r.cancelledPVB, pvb.Name) + } + + return ctrl.Result{}, nil + } + } + } + + if pvb.Status.Phase == "" || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseNew { + if pvb.Spec.Cancel { + log.Infof("PVB %s is canceled in Phase %s", pvb.GetName(), pvb.Status.Phase) + r.tryCancelPodVolumeBackup(ctx, pvb, "") + + return ctrl.Result{}, nil + } + + // Only process items for this node. + if pvb.Spec.Node != r.nodeName { + return ctrl.Result{}, nil + } + + log.Info("Accepting PVB") + + if err := r.acceptPodVolumeBackup(ctx, pvb); err != nil { + return ctrl.Result{}, errors.Wrapf(err, "error accepting PVB %s", pvb.Name) + } + + log.Info("Exposing PVB") + + exposeParam := r.setupExposeParam(pvb) + if err := r.exposer.Expose(ctx, getPVBOwnerObject(pvb), exposeParam); err != nil { + return r.errorOut(ctx, pvb, err, "error to expose PVB", log) + } + + log.Info("PVB is exposed") + return ctrl.Result{}, nil - } + } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted { + if peekErr := r.exposer.PeekExposed(ctx, getPVBOwnerObject(pvb)); peekErr != nil { + log.Errorf("Cancel PVB %s/%s because of expose error %s", pvb.Namespace, pvb.Name, peekErr) + r.tryCancelPodVolumeBackup(ctx, pvb, fmt.Sprintf("found a PVB %s/%s with expose error: %s. mark it as cancel", pvb.Namespace, pvb.Name, peekErr)) + } else if pvb.Status.AcceptedTimestamp != nil { + if time.Since(pvb.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { + r.onPrepareTimeout(ctx, pvb) + } + } - switch pvb.Status.Phase { - case "", velerov1api.PodVolumeBackupPhaseNew: - // Only process new items. - default: - log.Debug("PodVolumeBackup is not new, not processing") return ctrl.Result{}, nil - } + } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhasePrepared { + log.Infof("PVB is prepared and should be processed by %s (%s)", pvb.Spec.Node, r.nodeName) - log.Info("PodVolumeBackup starting") + if pvb.Spec.Node != r.nodeName { + return ctrl.Result{}, nil + } - callbacks := datapath.Callbacks{ - OnCompleted: r.OnDataPathCompleted, - OnFailed: r.OnDataPathFailed, - OnCancelled: r.OnDataPathCancelled, - OnProgress: r.OnDataPathProgress, - } + if pvb.Spec.Cancel { + log.Info("Prepared PVB is being canceled") + r.OnDataPathCancelled(ctx, pvb.GetNamespace(), pvb.GetName()) + return ctrl.Result{}, nil + } - fsBackup, err := r.dataPathMgr.CreateFileSystemBR(pvb.Name, pVBRRequestor, ctx, r.Client, pvb.Namespace, callbacks, log) + asyncBR := r.dataPathMgr.GetAsyncBR(pvb.Name) + if asyncBR != nil { + log.Info("Cancellable data path is already started") + return ctrl.Result{}, nil + } - if err != nil { - if err == datapath.ConcurrentLimitExceed { + res, err := r.exposer.GetExposed(ctx, getPVBOwnerObject(pvb), r.client, r.nodeName, r.resourceTimeout) + if err != nil { + return r.errorOut(ctx, pvb, err, "exposed PVB is not ready", log) + } else if res == nil { + return r.errorOut(ctx, pvb, errors.New("no expose result is available for the current node"), "exposed PVB is not ready", log) + } + + log.Info("Exposed PVB is ready and creating data path routine") + + callbacks := datapath.Callbacks{ + OnCompleted: r.OnDataPathCompleted, + OnFailed: r.OnDataPathFailed, + OnCancelled: r.OnDataPathCancelled, + OnProgress: r.OnDataPathProgress, + } + + asyncBR, err = r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeBackup, + pvb.Name, pvb.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, pvb.Name, callbacks, false, log) + if err != nil { + if err == datapath.ConcurrentLimitExceed { + log.Info("Data path instance is concurrent limited requeue later") + return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + } else { + return r.errorOut(ctx, pvb, err, "error to create data path", log) + } + } + + r.metrics.RegisterPodVolumeBackupEnqueue(r.nodeName) + + if err := r.initCancelableDataPath(ctx, asyncBR, res, log); err != nil { + log.WithError(err).Errorf("Failed to init cancelable data path for %s", pvb.Name) + + r.closeDataPath(ctx, pvb.Name) + return r.errorOut(ctx, pvb, err, "error initializing data path", log) + } + + terminated := false + if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { + if isPVBInFinalState(pvb) { + terminated = true + return false + } + + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseInProgress + pvb.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + + return true + }); err != nil { + log.WithError(err).Warnf("Failed to update PVB %s to InProgress, will data path close and retry", pvb.Name) + + r.closeDataPath(ctx, pvb.Name) return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil - } else { - return r.errorOut(ctx, &pvb, err, "error to create data path", log) } - } - r.metrics.RegisterPodVolumeBackupEnqueue(r.nodeName) - - // Update status to InProgress. - original := pvb.DeepCopy() - pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseInProgress - pvb.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} - if err := r.Client.Patch(ctx, &pvb, client.MergeFrom(original)); err != nil { - r.closeDataPath(ctx, pvb.Name) - return r.errorOut(ctx, &pvb, err, "error updating PodVolumeBackup status", log) - } - - var pod corev1api.Pod - podNamespacedName := client.ObjectKey{ - Namespace: pvb.Spec.Pod.Namespace, - Name: pvb.Spec.Pod.Name, - } - if err := r.Client.Get(ctx, podNamespacedName, &pod); err != nil { - r.closeDataPath(ctx, pvb.Name) - return r.errorOut(ctx, &pvb, err, fmt.Sprintf("getting pod %s/%s", pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name), log) - } - - path, err := exposer.GetPodVolumeHostPath(ctx, &pod, pvb.Spec.Volume, r.kubeClient, r.fileSystem, log) - if err != nil { - r.closeDataPath(ctx, pvb.Name) - return r.errorOut(ctx, &pvb, err, "error exposing host path for pod volume", log) - } - - log.WithField("path", path.ByPath).Debugf("Found host path") - - if err := fsBackup.Init(ctx, &datapath.FSBRInitParam{ - BSLName: pvb.Spec.BackupStorageLocation, - SourceNamespace: pvb.Spec.Pod.Namespace, - UploaderType: pvb.Spec.UploaderType, - RepositoryType: podvolume.GetPvbRepositoryType(&pvb), - RepoIdentifier: pvb.Spec.RepoIdentifier, - RepositoryEnsurer: r.repositoryEnsurer, - CredentialGetter: r.credentialGetter, - }); err != nil { - r.closeDataPath(ctx, pvb.Name) - return r.errorOut(ctx, &pvb, err, "error to initialize data path", log) - } - - // If this is a PVC, look for the most recent completed pod volume backup for it and get - // its snapshot ID to do new backup based on it. Without this, - // if the pod using the PVC (and therefore the directory path under /host_pods/) has - // changed since the PVC's last backup, for backup, it will not be able to identify a suitable - // parent snapshot to use, and will have to do a full rescan of the contents of the PVC. - var parentSnapshotID string - if pvcUID, ok := pvb.Labels[velerov1api.PVCUIDLabel]; ok { - parentSnapshotID = r.getParentSnapshot(ctx, log, pvcUID, &pvb) - if parentSnapshotID == "" { - log.Info("No parent snapshot found for PVC, not based on parent snapshot for this backup") - } else { - log.WithField("parentSnapshotID", parentSnapshotID).Info("Based on parent snapshot for this backup") + if terminated { + log.Warnf("PVB %s is terminated during transition from prepared", pvb.Name) + r.closeDataPath(ctx, pvb.Name) + return ctrl.Result{}, nil } - } - if err := fsBackup.StartBackup(path, pvb.Spec.UploaderSettings, &datapath.FSBRStartParam{ - RealSource: "", - ParentSnapshot: parentSnapshotID, - ForceFull: false, - Tags: pvb.Spec.Tags, - }); err != nil { - r.closeDataPath(ctx, pvb.Name) - return r.errorOut(ctx, &pvb, err, "error starting data path backup", log) - } + log.Info("PVB is marked as in progress") - log.WithField("path", path.ByPath).Info("Async fs backup data path started") + if err := r.startCancelableDataPath(asyncBR, pvb, res, log); err != nil { + log.WithError(err).Errorf("Failed to start cancelable data path for %s", pvb.Name) + r.closeDataPath(ctx, pvb.Name) + + return r.errorOut(ctx, pvb, err, "error starting data path", log) + } + + return ctrl.Result{}, nil + } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { + if pvb.Spec.Cancel { + if pvb.Spec.Node != r.nodeName { + return ctrl.Result{}, nil + } + + log.Info("In progress PVB is being canceled") + + asyncBR := r.dataPathMgr.GetAsyncBR(pvb.Name) + if asyncBR == nil { + r.OnDataPathCancelled(ctx, pvb.GetNamespace(), pvb.GetName()) + return ctrl.Result{}, nil + } + + // Update status to Canceling + if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { + if isPVBInFinalState(pvb) { + log.Warnf("PVB %s is terminated, abort setting it to canceling", pvb.Name) + return false + } + + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceling + return true + }); err != nil { + log.WithError(err).Error("error updating PVB into canceling status") + return ctrl.Result{}, err + } + + asyncBR.Cancel() + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil + } return ctrl.Result{}, nil } +func (r *PodVolumeBackupReconciler) acceptPodVolumeBackup(ctx context.Context, pvb *velerov1api.PodVolumeBackup) error { + return UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, r.logger, func(pvb *velerov1api.PodVolumeBackup) bool { + pvb.Status.AcceptedTimestamp = &metav1.Time{Time: r.clock.Now()} + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseAccepted + + return true + }) +} + +func (r *PodVolumeBackupReconciler) tryCancelPodVolumeBackup(ctx context.Context, pvb *velerov1api.PodVolumeBackup, message string) bool { + log := r.logger.WithField("PVB", pvb.Name) + succeeded, err := funcExclusiveUpdatePodVolumeBackup(ctx, r.client, pvb, func(pvb *velerov1api.PodVolumeBackup) { + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceled + if pvb.Status.StartTimestamp.IsZero() { + pvb.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + } + pvb.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} + + if message != "" { + pvb.Status.Message = message + } + }) + + if err != nil { + log.WithError(err).Error("error updating PVB status") + return false + } else if !succeeded { + log.Warn("conflict in updating PVB status and will try it again later") + return false + } + + r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) + + log.Warn("PVB is canceled") + + return true +} + +var funcExclusiveUpdatePodVolumeBackup = exclusiveUpdatePodVolumeBackup + +func exclusiveUpdatePodVolumeBackup(ctx context.Context, cli client.Client, pvb *velerov1api.PodVolumeBackup, updateFunc func(*velerov1api.PodVolumeBackup)) (bool, error) { + updateFunc(pvb) + + err := cli.Update(ctx, pvb) + if err == nil { + return true, nil + } + + if apierrors.IsConflict(err) { + return false, nil + } else { + return false, err + } +} + +func (r *PodVolumeBackupReconciler) onPrepareTimeout(ctx context.Context, pvb *velerov1api.PodVolumeBackup) { + log := r.logger.WithField("PVB", pvb.Name) + + log.Info("Timeout happened for preparing PVB") + + succeeded, err := funcExclusiveUpdatePodVolumeBackup(ctx, r.client, pvb, func(pvb *velerov1api.PodVolumeBackup) { + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed + pvb.Status.Message = "timeout on preparing PVB" + }) + + if err != nil { + log.WithError(err).Warn("Failed to update PVB") + return + } + + if !succeeded { + log.Warn("PVB has been updated by others") + return + } + + diags := strings.Split(r.exposer.DiagnoseExpose(ctx, getPVBOwnerObject(pvb)), "\n") + for _, diag := range diags { + log.Warnf("[Diagnose PVB expose]%s", diag) + } + + r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) + + log.Info("PVB has been cleaned up") +} + +func (r *PodVolumeBackupReconciler) initCancelableDataPath(ctx context.Context, asyncBR datapath.AsyncBR, res *exposer.ExposeResult, log logrus.FieldLogger) error { + log.Info("Init cancelable PVB") + + if err := asyncBR.Init(ctx, nil); err != nil { + return errors.Wrap(err, "error initializing asyncBR") + } + + log.Infof("async data path init for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) + + return nil +} + +func (r *PodVolumeBackupReconciler) startCancelableDataPath(asyncBR datapath.AsyncBR, pvb *velerov1api.PodVolumeBackup, res *exposer.ExposeResult, log logrus.FieldLogger) error { + log.Info("Start cancelable PVB") + + if err := asyncBR.StartBackup(datapath.AccessPoint{ + ByPath: res.ByPod.VolumeName, + }, pvb.Spec.UploaderSettings, nil); err != nil { + return errors.Wrapf(err, "error starting async backup for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) + } + + log.Infof("Async backup started for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) + return nil +} + func (r *PodVolumeBackupReconciler) OnDataPathCompleted(ctx context.Context, namespace string, pvbName string, result datapath.Result) { defer r.dataPathMgr.RemoveAsyncBR(pvbName) - log := r.logger.WithField("pvb", pvbName) + log := r.logger.WithField("PVB", pvbName) log.WithField("PVB", pvbName).Info("Async fs backup data path completed") - var pvb velerov1api.PodVolumeBackup - if err := r.Client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); err != nil { + pvb := &velerov1api.PodVolumeBackup{} + if err := r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, pvb); err != nil { log.WithError(err).Warn("Failed to get PVB on completion") return } + log.Info("Cleaning up exposed environment") + r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) + // Update status to Completed with path & snapshot ID. - original := pvb.DeepCopy() - pvb.Status.Path = result.Backup.Source.ByPath - pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCompleted - pvb.Status.SnapshotID = result.Backup.SnapshotID - pvb.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} - if result.Backup.EmptySnapshot { - pvb.Status.Message = "volume was empty so no snapshot was taken" + var completionTime metav1.Time + if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { + completionTime = metav1.Time{Time: r.clock.Now()} + + if isPVBInFinalState(pvb) { + return false + } + + pvb.Status.Path = result.Backup.Source.ByPath + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCompleted + pvb.Status.SnapshotID = result.Backup.SnapshotID + pvb.Status.CompletionTimestamp = &completionTime + if result.Backup.EmptySnapshot { + pvb.Status.Message = "volume was empty so no snapshot was taken" + } + + return true + }); err != nil { + log.WithError(err).Error("error updating PVB status") + } else { + latencyDuration := completionTime.Time.Sub(pvb.Status.StartTimestamp.Time) + latencySeconds := float64(latencyDuration / time.Second) + backupName := fmt.Sprintf("%s/%s", pvb.Namespace, pvb.OwnerReferences[0].Name) + generateOpName := fmt.Sprintf("%s-%s-%s-%s-backup", pvb.Name, pvb.Spec.BackupStorageLocation, pvb.Spec.Pod.Namespace, pvb.Spec.UploaderType) + r.metrics.ObservePodVolumeOpLatency(r.nodeName, pvb.Name, generateOpName, backupName, latencySeconds) + r.metrics.RegisterPodVolumeOpLatencyGauge(r.nodeName, pvb.Name, generateOpName, backupName, latencySeconds) + r.metrics.RegisterPodVolumeBackupDequeue(r.nodeName) + + log.Info("PVB completed") } - - if err := r.Client.Patch(ctx, &pvb, client.MergeFrom(original)); err != nil { - log.WithError(err).Error("error updating PodVolumeBackup status") - } - - latencyDuration := pvb.Status.CompletionTimestamp.Time.Sub(pvb.Status.StartTimestamp.Time) - latencySeconds := float64(latencyDuration / time.Second) - backupName := fmt.Sprintf("%s/%s", pvb.Namespace, pvb.OwnerReferences[0].Name) - generateOpName := fmt.Sprintf("%s-%s-%s-%s-backup", pvb.Name, pvb.Spec.BackupStorageLocation, pvb.Spec.Pod.Namespace, pvb.Spec.UploaderType) - r.metrics.ObservePodVolumeOpLatency(r.nodeName, pvb.Name, generateOpName, backupName, latencySeconds) - r.metrics.RegisterPodVolumeOpLatencyGauge(r.nodeName, pvb.Name, generateOpName, backupName, latencySeconds) - r.metrics.RegisterPodVolumeBackupDequeue(r.nodeName) - - log.Info("PodVolumeBackup completed") } func (r *PodVolumeBackupReconciler) OnDataPathFailed(ctx context.Context, namespace, pvbName string, err error) { defer r.dataPathMgr.RemoveAsyncBR(pvbName) - log := r.logger.WithField("pvb", pvbName) + log := r.logger.WithField("PVB", pvbName) log.WithError(err).Error("Async fs backup data path failed") var pvb velerov1api.PodVolumeBackup - if getErr := r.Client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); getErr != nil { + if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); getErr != nil { log.WithError(getErr).Warn("Failed to get PVB on failure") } else { _, _ = r.errorOut(ctx, &pvb, err, "data path backup failed", log) @@ -268,131 +542,305 @@ func (r *PodVolumeBackupReconciler) OnDataPathFailed(ctx context.Context, namesp func (r *PodVolumeBackupReconciler) OnDataPathCancelled(ctx context.Context, namespace string, pvbName string) { defer r.dataPathMgr.RemoveAsyncBR(pvbName) - log := r.logger.WithField("pvb", pvbName) + log := r.logger.WithField("PVB", pvbName) log.Warn("Async fs backup data path canceled") var pvb velerov1api.PodVolumeBackup - if getErr := r.Client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); getErr != nil { + if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); getErr != nil { log.WithError(getErr).Warn("Failed to get PVB on cancel") + return + } + // cleans up any objects generated during the snapshot expose + r.exposer.CleanUp(ctx, getPVBOwnerObject(&pvb)) + + if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, func(pvb *velerov1api.PodVolumeBackup) bool { + if isPVBInFinalState(pvb) { + return false + } + + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceled + if pvb.Status.StartTimestamp.IsZero() { + pvb.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + } + pvb.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} + + return true + }); err != nil { + log.WithError(err).Error("error updating PVB status on cancel") } else { - _, _ = r.errorOut(ctx, &pvb, errors.New("PVB is canceled"), "data path backup canceled", log) + delete(r.cancelledPVB, pvb.Name) } } func (r *PodVolumeBackupReconciler) OnDataPathProgress(ctx context.Context, namespace string, pvbName string, progress *uploader.Progress) { log := r.logger.WithField("pvb", pvbName) - var pvb velerov1api.PodVolumeBackup - if err := r.Client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, &pvb); err != nil { - log.WithError(err).Warn("Failed to get PVB on progress") - return - } - - original := pvb.DeepCopy() - pvb.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} - - if err := r.Client.Patch(ctx, &pvb, client.MergeFrom(original)); err != nil { + if err := UpdatePVBWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvbName}, log, func(pvb *velerov1api.PodVolumeBackup) bool { + pvb.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} + return true + }); err != nil { log.WithError(err).Error("Failed to update progress") } } // SetupWithManager registers the PVB controller. func (r *PodVolumeBackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + gp := kube.NewGenericEventPredicate(func(object client.Object) bool { + pvb := object.(*velerov1api.PodVolumeBackup) + if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted { + return true + } + + if pvb.Spec.Cancel && !isPVBInFinalState(pvb) { + return true + } + + if isPVBInFinalState(pvb) && !pvb.DeletionTimestamp.IsZero() { + return true + } + + return false + }) + + s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerPodVolumeBackup), r.client, &velerov1api.PodVolumeBackupList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{ + Predicates: []predicate.Predicate{gp}, + }) + return ctrl.NewControllerManagedBy(mgr). For(&velerov1api.PodVolumeBackup{}). + WatchesRawSource(s). + Watches(&corev1api.Pod{}, kube.EnqueueRequestsFromMapUpdateFunc(r.findPVBForPod), + builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + newObj := ue.ObjectNew.(*corev1api.Pod) + + if _, ok := newObj.Labels[velerov1api.PVBLabel]; !ok { + return false + } + + if newObj.Spec.NodeName == "" { + return false + } + + return true + }, + CreateFunc: func(event.CreateEvent) bool { + return false + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). Complete(r) } -// getParentSnapshot finds the most recent completed PodVolumeBackup for the -// specified PVC and returns its snapshot ID. Any errors encountered are -// logged but not returned since they do not prevent a backup from proceeding. -func (r *PodVolumeBackupReconciler) getParentSnapshot(ctx context.Context, log logrus.FieldLogger, pvcUID string, podVolumeBackup *velerov1api.PodVolumeBackup) string { - log = log.WithField("pvcUID", pvcUID) - log.Infof("Looking for most recent completed PodVolumeBackup for this PVC") +func (r *PodVolumeBackupReconciler) findPVBForPod(ctx context.Context, podObj client.Object) []reconcile.Request { + pod := podObj.(*corev1api.Pod) + pvb, err := findPVBByPod(r.client, *pod) - listOpts := &client.ListOptions{ - Namespace: podVolumeBackup.Namespace, + log := r.logger.WithField("pod", pod.Name) + if err != nil { + log.WithError(err).Error("unable to get PVB") + return []reconcile.Request{} + } else if pvb == nil { + log.Error("get empty PVB") + return []reconcile.Request{} } - matchingLabels := client.MatchingLabels(map[string]string{velerov1api.PVCUIDLabel: pvcUID}) - matchingLabels.ApplyToList(listOpts) + log = log.WithFields(logrus.Fields{ + "PVB": pvb.Name, + }) - var pvbList velerov1api.PodVolumeBackupList - if err := r.Client.List(ctx, &pvbList, listOpts); err != nil { - log.WithError(errors.WithStack(err)).Error("getting list of podvolumebackups for this PVC") + if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseAccepted { + return []reconcile.Request{} } - // Go through all the podvolumebackups for the PVC and look for the most - // recent completed one to use as the parent. - var mostRecentPVB velerov1api.PodVolumeBackup - for _, pvb := range pvbList.Items { - if pvb.Spec.UploaderType != podVolumeBackup.Spec.UploaderType { - continue - } - if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted { - continue + if pod.Status.Phase == corev1api.PodRunning { + log.Info("Preparing PVB") + + if err = UpdatePVBWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, + func(pvb *velerov1api.PodVolumeBackup) bool { + if isPVBInFinalState(pvb) { + log.Warnf("PVB %s is terminated, abort setting it to prepared", pvb.Name) + return false + } + + pvb.Status.Phase = velerov1api.PodVolumeBackupPhasePrepared + return true + }); err != nil { + log.WithError(err).Warn("Failed to update PVB, prepare will halt for this PVB") + return []reconcile.Request{} } + } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { + err := UpdatePVBWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, + func(pvb *velerov1api.PodVolumeBackup) bool { + if pvb.Spec.Cancel { + return false + } - if podVolumeBackup.Spec.BackupStorageLocation != pvb.Spec.BackupStorageLocation { - // Check the backup storage location is the same as spec in order to - // support backup to multiple backup-locations. Otherwise, there exists - // a case that backup volume snapshot to the second location would - // failed, since the founded parent ID is only valid for the first - // backup location, not the second backup location. Also, the second - // backup should not use the first backup parent ID since its for the - // first backup location only. - continue - } + pvb.Spec.Cancel = true + pvb.Status.Message = fmt.Sprintf("Cancel PVB because the exposing pod %s/%s is in abnormal status for reason %s", pod.Namespace, pod.Name, reason) - if mostRecentPVB.Status == (velerov1api.PodVolumeBackupStatus{}) || pvb.Status.StartTimestamp.After(mostRecentPVB.Status.StartTimestamp.Time) { - mostRecentPVB = pvb + return true + }) + + if err != nil { + log.WithError(err).Warn("failed to cancel PVB, and it will wait for prepare timeout") + return []reconcile.Request{} } + log.Infof("Exposed pod is in abnormal status(reason %s) and PVB is marked as cancel", reason) + } else { + return []reconcile.Request{} } - if mostRecentPVB.Status == (velerov1api.PodVolumeBackupStatus{}) { - log.Info("No completed PodVolumeBackup found for PVC") - return "" + request := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: pvb.Namespace, + Name: pvb.Name, + }, } - - log.WithFields(map[string]any{ - "parentPodVolumeBackup": mostRecentPVB.Name, - "parentSnapshotID": mostRecentPVB.Status.SnapshotID, - }).Info("Found most recent completed PodVolumeBackup for PVC") - - return mostRecentPVB.Status.SnapshotID -} - -func (r *PodVolumeBackupReconciler) closeDataPath(ctx context.Context, pvbName string) { - fsBackup := r.dataPathMgr.GetAsyncBR(pvbName) - if fsBackup != nil { - fsBackup.Close(ctx) - } - - r.dataPathMgr.RemoveAsyncBR(pvbName) + return []reconcile.Request{request} } func (r *PodVolumeBackupReconciler) errorOut(ctx context.Context, pvb *velerov1api.PodVolumeBackup, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { - _ = UpdatePVBStatusToFailed(ctx, r.Client, pvb, err, msg, r.clock.Now(), log) + r.exposer.CleanUp(ctx, getPVBOwnerObject(pvb)) + + _ = UpdatePVBStatusToFailed(ctx, r.client, pvb, err, msg, r.clock.Now(), log) return ctrl.Result{}, err } func UpdatePVBStatusToFailed(ctx context.Context, c client.Client, pvb *velerov1api.PodVolumeBackup, errOut error, msg string, time time.Time, log logrus.FieldLogger) error { - original := pvb.DeepCopy() - pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed - pvb.Status.CompletionTimestamp = &metav1.Time{Time: time} - if dataPathError, ok := errOut.(datapath.DataPathError); ok { - pvb.Status.SnapshotID = dataPathError.GetSnapshotID() - } - if len(strings.TrimSpace(msg)) == 0 { - pvb.Status.Message = errOut.Error() - } else { - pvb.Status.Message = errors.WithMessage(errOut, msg).Error() - } - err := c.Patch(ctx, pvb, client.MergeFrom(original)) - if err != nil { - log.WithError(err).Error("error updating PodVolumeBackup status") + log.Info("update PVB status to Failed") + + if patchErr := UpdatePVBWithRetry(context.Background(), c, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log, + func(pvb *velerov1api.PodVolumeBackup) bool { + if isPVBInFinalState(pvb) { + return false + } + + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed + pvb.Status.CompletionTimestamp = &metav1.Time{Time: time} + if dataPathError, ok := errOut.(datapath.DataPathError); ok { + pvb.Status.SnapshotID = dataPathError.GetSnapshotID() + } + if len(strings.TrimSpace(msg)) == 0 { + pvb.Status.Message = errOut.Error() + } else { + pvb.Status.Message = errors.WithMessage(errOut, msg).Error() + } + if pvb.Status.StartTimestamp.IsZero() { + pvb.Status.StartTimestamp = &metav1.Time{Time: time} + } + + return true + }); patchErr != nil { + log.WithError(patchErr).Warn("error updating PVB status") } - return err + return errOut +} + +func (r *PodVolumeBackupReconciler) closeDataPath(ctx context.Context, pvbName string) { + asyncBR := r.dataPathMgr.GetAsyncBR(pvbName) + if asyncBR != nil { + asyncBR.Close(ctx) + } + + r.dataPathMgr.RemoveAsyncBR(pvbName) +} + +func (r *PodVolumeBackupReconciler) setupExposeParam(pvb *velerov1api.PodVolumeBackup) exposer.PodVolumeExposeParam { + log := r.logger.WithField("PVB", pvb.Name) + + hostingPodLabels := map[string]string{velerov1api.PVBLabel: pvb.Name} + for _, k := range util.ThirdPartyLabels { + if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, pvb.Namespace, k, ""); err != nil { + if err != nodeagent.ErrNodeAgentLabelNotFound { + log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) + } + } else { + hostingPodLabels[k] = v + } + } + + hostingPodAnnotation := map[string]string{} + for _, k := range util.ThirdPartyAnnotations { + if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, pvb.Namespace, k, ""); err != nil { + if err != nodeagent.ErrNodeAgentAnnotationNotFound { + log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) + } + } else { + hostingPodAnnotation[k] = v + } + } + + return exposer.PodVolumeExposeParam{ + Type: exposer.PodVolumeExposeTypeBackup, + ClientNamespace: pvb.Spec.Pod.Namespace, + ClientPodName: pvb.Spec.Pod.Name, + ClientPodVolume: pvb.Spec.Volume, + HostingPodLabels: hostingPodLabels, + HostingPodAnnotations: hostingPodAnnotation, + OperationTimeout: r.resourceTimeout, + Resources: r.podResources, + } +} + +func getPVBOwnerObject(pvb *velerov1api.PodVolumeBackup) corev1api.ObjectReference { + return corev1api.ObjectReference{ + Kind: pvb.Kind, + Namespace: pvb.Namespace, + Name: pvb.Name, + UID: pvb.UID, + APIVersion: pvb.APIVersion, + } +} + +func findPVBByPod(client client.Client, pod corev1api.Pod) (*velerov1api.PodVolumeBackup, error) { + if label, exist := pod.Labels[velerov1api.PVBLabel]; exist { + pvb := &velerov1api.PodVolumeBackup{} + err := client.Get(context.Background(), types.NamespacedName{ + Namespace: pod.Namespace, + Name: label, + }, pvb) + + if err != nil { + return nil, errors.Wrapf(err, "error to find PVB by pod %s/%s", pod.Namespace, pod.Name) + } + return pvb, nil + } + return nil, nil +} + +func isPVBInFinalState(pvb *velerov1api.PodVolumeBackup) bool { + return pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || + pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled || + pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted +} + +func UpdatePVBWithRetry(ctx context.Context, client client.Client, namespacedName types.NamespacedName, log logrus.FieldLogger, updateFunc func(*velerov1api.PodVolumeBackup) bool) error { + return wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (bool, error) { + pvb := &velerov1api.PodVolumeBackup{} + if err := client.Get(ctx, namespacedName, pvb); err != nil { + return false, errors.Wrap(err, "getting PVB") + } + + if updateFunc(pvb) { + err := client.Update(ctx, pvb) + if err != nil { + if apierrors.IsConflict(err) { + log.Warnf("failed to update PVB for %s/%s and will retry it", pvb.Namespace, pvb.Name) + return false, nil + } else { + return false, errors.Wrapf(err, "error updating PVB with error %s/%s", pvb.Namespace, pvb.Name) + } + } + } + + return true, nil + }) } diff --git a/pkg/controller/pod_volume_backup_controller_test.go b/pkg/controller/pod_volume_backup_controller_test.go index d5f1e3471..38249bfa0 100644 --- a/pkg/controller/pod_volume_backup_controller_test.go +++ b/pkg/controller/pod_volume_backup_controller_test.go @@ -19,183 +19,487 @@ package controller import ( "context" "fmt" + "testing" "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/kubernetes" + clientgofake "k8s.io/client-go/kubernetes/fake" "k8s.io/utils/clock" - testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" - kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/datapath" + "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/uploader" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) -const name = "pvb-1" +const pvbName = "pvb-1" + +func initPVBReconciler(needError ...bool) (*PodVolumeBackupReconciler, error) { + var errs = make([]error, 6) + for k, isError := range needError { + if k == 0 && isError { + errs[0] = fmt.Errorf("Get error") + } else if k == 1 && isError { + errs[1] = fmt.Errorf("Create error") + } else if k == 2 && isError { + errs[2] = fmt.Errorf("Update error") + } else if k == 3 && isError { + errs[3] = fmt.Errorf("Patch error") + } else if k == 4 && isError { + errs[4] = apierrors.NewConflict(velerov1api.Resource("podvolumebackup"), pvbName, errors.New("conflict")) + } else if k == 5 && isError { + errs[5] = fmt.Errorf("List error") + } + } + + return initPVBReconcilerWithError(errs...) +} + +func initPVBReconcilerWithError(needError ...error) (*PodVolumeBackupReconciler, error) { + daemonSet := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "node-agent", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: appsv1api.SchemeGroupVersion.String(), + }, + Spec: appsv1api.DaemonSetSpec{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{ + Containers: []corev1api.Container{ + { + Image: "fake-image", + }, + }, + }, + }, + }, + } + + node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() + + dataPathMgr := datapath.NewManager(1) + + scheme := runtime.NewScheme() + err := velerov1api.AddToScheme(scheme) + if err != nil { + return nil, err + } + + err = corev1api.AddToScheme(scheme) + if err != nil { + return nil, err + } + + fakeClient := &FakeClient{ + Client: fake.NewClientBuilder().WithScheme(scheme).Build(), + } + + for k := range needError { + if k == 0 { + fakeClient.getError = needError[0] + } else if k == 1 { + fakeClient.createError = needError[1] + } else if k == 2 { + fakeClient.updateError = needError[2] + } else if k == 3 { + fakeClient.patchError = needError[3] + } else if k == 4 { + fakeClient.updateConflict = needError[4] + } else if k == 5 { + fakeClient.listError = needError[5] + } + } + + fakeKubeClient := clientgofake.NewSimpleClientset(daemonSet, node) + + return NewPodVolumeBackupReconciler( + fakeClient, + nil, + fakeKubeClient, + dataPathMgr, + "test-node", + time.Minute*5, + time.Minute, + corev1api.ResourceRequirements{}, + metrics.NewServerMetrics(), + velerotest.NewLogger(), + ), nil +} func pvbBuilder() *builder.PodVolumeBackupBuilder { - return builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, name). - PodNamespace(velerov1api.DefaultNamespace). - PodName(name). - Volume("pvb-1-volume"). - BackupStorageLocation("bsl-loc"). - ObjectMeta( - func(obj metav1.Object) { - obj.SetOwnerReferences([]metav1.OwnerReference{{Name: name}}) - }, - ) + return builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName).BackupStorageLocation("bsl-loc") } -func podBuilder() *builder.PodBuilder { - return builder. - ForPod(velerov1api.DefaultNamespace, name). - Volumes(&corev1api.Volume{Name: "pvb-1-volume"}) +type fakePvbExposer struct { + kubeClient client.Client + clock clock.WithTickerAndDelayedExecution + peekErr error + exposeErr error + getErr error + getNil bool } -func bslBuilder() *builder.BackupStorageLocationBuilder { - return builder. - ForBackupStorageLocation(velerov1api.DefaultNamespace, "bsl-loc") -} - -func buildBackupRepo() *velerov1api.BackupRepository { - return &velerov1api.BackupRepository{ - Spec: velerov1api.BackupRepositorySpec{ResticIdentifier: ""}, - TypeMeta: metav1.TypeMeta{ - APIVersion: velerov1api.SchemeGroupVersion.String(), - Kind: "BackupRepository", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: velerov1api.DefaultNamespace, - Name: fmt.Sprintf("%s-bsl-loc-restic-dn24h", velerov1api.DefaultNamespace), - Labels: map[string]string{ - velerov1api.StorageLocationLabel: "bsl-loc", - velerov1api.VolumeNamespaceLabel: velerov1api.DefaultNamespace, - velerov1api.RepositoryTypeLabel: "restic", - }, - }, - Status: velerov1api.BackupRepositoryStatus{ - Phase: velerov1api.BackupRepositoryPhaseReady, - }, - } -} - -type fakeFSBR struct { - pvb *velerov1api.PodVolumeBackup - client kbclient.Client - clock clock.WithTickerAndDelayedExecution -} - -func (b *fakeFSBR) Init(ctx context.Context, param any) error { - return nil -} - -func (b *fakeFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[string]string, param any) error { - pvb := b.pvb - - original := b.pvb.DeepCopy() - pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCompleted - pvb.Status.CompletionTimestamp = &metav1.Time{Time: b.clock.Now()} - - b.client.Patch(ctx, pvb, kbclient.MergeFrom(original)) - - return nil -} - -func (b *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error { - return nil -} - -func (b *fakeFSBR) Cancel() { -} - -func (b *fakeFSBR) Close(ctx context.Context) { -} - -var _ = Describe("PodVolumeBackup Reconciler", func() { - type request struct { - pvb *velerov1api.PodVolumeBackup - pod *corev1api.Pod - bsl *velerov1api.BackupStorageLocation - backupRepo *velerov1api.BackupRepository - expectedProcessed bool - expected *velerov1api.PodVolumeBackup - expectedRequeue ctrl.Result - expectedErrMsg string - dataMgr *datapath.Manager +func (f *fakePvbExposer) Expose(ctx context.Context, ownerObject corev1api.ObjectReference, param exposer.PodVolumeExposeParam) error { + if f.exposeErr != nil { + return f.exposeErr } - // `now` will be used to set the fake clock's time; capture - // it here so it can be referenced in the test case defs. - now, err := time.Parse(time.RFC1123, time.RFC1123) - Expect(err).ToNot(HaveOccurred()) - now = now.Local() + return nil +} - DescribeTable("a pod volume backup", - func(test request) { - ctx := context.Background() +func (f *fakePvbExposer) GetExposed(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) (*exposer.ExposeResult, error) { + if f.getErr != nil { + return nil, f.getErr + } - fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() - err = fakeClient.Create(ctx, test.pvb) - Expect(err).ToNot(HaveOccurred()) + if f.getNil { + return nil, nil + } - err = fakeClient.Create(ctx, test.pod) - Expect(err).ToNot(HaveOccurred()) + pod := &corev1api.Pod{} - err = fakeClient.Create(ctx, test.bsl) - Expect(err).ToNot(HaveOccurred()) + nodeOS := "linux" + pNodeOS := &nodeOS - err = fakeClient.Create(ctx, test.backupRepo) - Expect(err).ToNot(HaveOccurred()) + return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: pvbName, NodeOS: pNodeOS}}, nil +} - fakeFS := velerotest.NewFakeFileSystem() - pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", "", "pvb-1-volume") - _, err = fakeFS.Create(pathGlob) - Expect(err).ToNot(HaveOccurred()) +func (f *fakePvbExposer) PeekExposed(ctx context.Context, ownerObject corev1api.ObjectReference) error { + return f.peekErr +} - credentialFileStore, err := credentials.NewNamespacedFileStore( - fakeClient, - velerov1api.DefaultNamespace, - "/tmp/credentials", - fakeFS, - ) +func (f *fakePvbExposer) DiagnoseExpose(context.Context, corev1api.ObjectReference) string { + return "" +} - Expect(err).ToNot(HaveOccurred()) +func (f *fakePvbExposer) CleanUp(context.Context, corev1api.ObjectReference) { +} - if test.dataMgr == nil { - test.dataMgr = datapath.NewManager(1) +func TestPVBReconcile(t *testing.T) { + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + notCreatePvb bool + needDelete bool + sportTime *metav1.Time + pod *corev1api.Pod + dataMgr *datapath.Manager + needCreateFSBR bool + needExclusiveUpdateError error + needMockExposer bool + expected *velerov1api.PodVolumeBackup + expectDeleted bool + expectCancelRecord bool + needErrs []bool + peekErr error + exposeErr error + getExposeErr error + getExposeNil bool + fsBRInitErr error + fsBRStartErr error + expectedErr string + expectedResult *ctrl.Result + expectDataPath bool + }{ + { + name: "pvb not found", + pvb: pvbBuilder().Result(), + notCreatePvb: true, + }, + { + name: "pvb not created in velero default namespace", + pvb: builder.ForPodVolumeBackup("test-ns", pvbName).Result(), + }, + { + name: "get pvb fail", + pvb: pvbBuilder().Result(), + needErrs: []bool{true, false, false, false}, + expectedErr: "getting PVB: Get error", + }, + { + name: "add finalizer to pvb", + pvb: pvbBuilder().Result(), + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "add finalizer to pvb failed", + pvb: pvbBuilder().Result(), + needErrs: []bool{false, false, true, false}, + expectedErr: "error updating PVB with error velero/pvb-1: Update error", + }, + { + name: "pvb is under deletion", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), + needDelete: true, + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(), + }, + { + name: "pvb is under deletion but cancel failed", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), + needErrs: []bool{false, false, true, false}, + needDelete: true, + expectedErr: "error updating PVB with error velero/pvb-1: Update error", + }, + { + name: "pvb is under deletion and in terminal state", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Result(), + sportTime: &metav1.Time{Time: time.Now()}, + needDelete: true, + expectDeleted: true, + }, + { + name: "pvb is under deletion and in terminal state, but remove finalizer failed", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Result(), + needErrs: []bool{false, false, true, false}, + needDelete: true, + expectedErr: "error updating PVB with error velero/pvb-1: Update error", + }, + { + name: "delay cancel negative for others", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhasePrepared).Result(), + sportTime: &metav1.Time{Time: time.Now()}, + expectCancelRecord: true, + }, + { + name: "delay cancel negative for inProgress", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), + sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 58)}, + expectCancelRecord: true, + }, + { + name: "delay cancel affirmative for others", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhasePrepared).Result(), + sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 5)}, + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), + }, + { + name: "delay cancel affirmative for inProgress", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), + sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), + }, + { + name: "delay cancel failed", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), + needErrs: []bool{false, false, true, false}, + sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), + expectCancelRecord: true, + }, + { + name: "Unknown pvb status", + pvb: pvbBuilder().Phase("Unknown").Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "new pvb but accept failed", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needErrs: []bool{false, false, true, false}, + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), + expectedErr: "error accepting PVB pvb-1: error updating PVB with error velero/pvb-1: Update error", + }, + { + name: "pvb is cancel on accepted", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Cancel(true).Result(), + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), + expectCancelRecord: true, + }, + { + name: "pvb expose failed", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + exposeErr: errors.New("fake-expose-error"), + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Message("error to expose PVB").Result(), + expectedErr: "fake-expose-error", + }, + { + name: "pvb succeeds for accepted", + pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), + }, + { + name: "prepare timeout on accepted", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseFailed).Message("timeout on preparing PVB").Result(), + }, + { + name: "peek error on accepted", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).Result(), + needMockExposer: true, + peekErr: errors.New("fake-peak-error"), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Message("found a PVB velero/pvb-1 with expose error: fake-peak-error. mark it as cancel").Result(), + }, + { + name: "cancel on prepared", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Cancel(true).Result(), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseCanceled).Result(), + }, + { + name: "Failed to get pvb expose on prepared", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + getExposeErr: errors.New("fake-get-error"), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVB is not ready: fake-get-error").Result(), + expectedErr: "fake-get-error", + }, + { + name: "Get nil restore expose on prepared", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + getExposeNil: true, + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVB is not ready").Result(), + expectedErr: "no expose result is available for the current node", + }, + { + name: "Error in data path is concurrent limited", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + dataMgr: datapath.NewManager(0), + expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + }, + { + name: "data path init error", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + fsBRInitErr: errors.New("fake-data-path-init-error"), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error initializing data path").Result(), + expectedErr: "error initializing asyncBR: fake-data-path-init-error", + }, + { + name: "Unable to update status to in progress for data upload", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + needErrs: []bool{false, false, true, false}, + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Result(), + expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + }, + { + name: "data path start error", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + fsBRStartErr: errors.New("fake-data-path-start-error"), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error starting data path").Result(), + expectedErr: "error starting async backup for pod , volume pvb-1: fake-data-path-start-error", + }, + { + name: "Prepare succeeds", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needMockExposer: true, + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + expectDataPath: true, + }, + { + name: "In progress pvb is not handled by the current node", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "In progress pvb is not set as cancel", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "Cancel pvb in progress with empty FSBR", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceled).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "Cancel pvb in progress and patch pvb error", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needErrs: []bool{false, false, true, false}, + needCreateFSBR: true, + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), + expectedErr: "error updating PVB with error velero/pvb-1: Update error", + expectCancelRecord: true, + expectDataPath: true, + }, + { + name: "Cancel pvb in progress succeeds", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needCreateFSBR: true, + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseCanceling).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), + expectDataPath: true, + expectCancelRecord: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r, err := initPVBReconciler(test.needErrs...) + require.NoError(t, err) + + if !test.notCreatePvb { + err = r.client.Create(context.Background(), test.pvb) + require.NoError(t, err) } - datapath.FSBRCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { - return &fakeFSBR{ - pvb: test.pvb, - client: fakeClient, - clock: testclocks.NewFakeClock(now), + if test.needDelete { + err = r.client.Delete(context.Background(), test.pvb) + require.NoError(t, err) + } + + if test.pod != nil { + err = r.client.Create(ctx, test.pod) + require.NoError(t, err) + } + + if test.dataMgr != nil { + r.dataPathMgr = test.dataMgr + } else { + r.dataPathMgr = datapath.NewManager(1) + } + + if test.sportTime != nil { + r.cancelledPVB[test.pvb.Name] = test.sportTime.Time + } + + if test.needMockExposer { + r.exposer = &fakePvbExposer{r.client, r.clock, test.peekErr, test.exposeErr, test.getExposeErr, test.getExposeNil} + } + + funcExclusiveUpdatePodVolumeBackup = exclusiveUpdatePodVolumeBackup + if test.needExclusiveUpdateError != nil { + funcExclusiveUpdatePodVolumeBackup = func(context.Context, client.Client, *velerov1api.PodVolumeBackup, func(*velerov1api.PodVolumeBackup)) (bool, error) { + return false, test.needExclusiveUpdateError } } - // Setup reconciler - Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed()) - r := PodVolumeBackupReconciler{ - Client: fakeClient, - clock: testclocks.NewFakeClock(now), - metrics: metrics.NewNodeMetrics(), - credentialGetter: &credentials.CredentialGetter{FromFile: credentialFileStore}, - nodeName: "test_node", - fileSystem: fakeFS, - logger: velerotest.NewLogger(), - dataPathMgr: test.dataMgr, + datapath.MicroServiceBRWatcherCreator = func(client.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { + return &fakeFSBR{ + kubeClient: r.client, + clock: r.clock, + initErr: test.fsBRInitErr, + startErr: test.fsBRStartErr, + } + } + + if test.needCreateFSBR { + if fsBR := r.dataPathMgr.GetAsyncBR(test.pvb.Name); fsBR == nil { + _, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, nil, nil, datapath.TaskTypeBackup, test.pvb.Name, velerov1api.DefaultNamespace, "", "", "", datapath.Callbacks{OnCancelled: r.OnDataPathCancelled}, false, velerotest.NewLogger()) + require.NoError(t, err) + } } actualResult, err := r.Reconcile(ctx, ctrl.Request{ @@ -204,184 +508,413 @@ var _ = Describe("PodVolumeBackup Reconciler", func() { Name: test.pvb.Name, }, }) - Expect(actualResult).To(BeEquivalentTo(test.expectedRequeue)) - if test.expectedErrMsg == "" { - Expect(err).ToNot(HaveOccurred()) + + if test.expectedErr != "" { + assert.EqualError(t, err, test.expectedErr) } else { - Expect(err.Error()).To(BeEquivalentTo(test.expectedErrMsg)) + assert.NoError(t, err) } - pvb := velerov1api.PodVolumeBackup{} - err = r.Client.Get(ctx, kbclient.ObjectKey{ - Name: test.pvb.Name, - Namespace: test.pvb.Namespace, - }, &pvb) - // Assertions - if test.expected == nil { - Expect(apierrors.IsNotFound(err)).To(BeTrue()) + if test.expectedResult != nil { + assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) + assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) + } + + if test.expected != nil || test.expectDeleted { + pvb := velerov1api.PodVolumeBackup{} + err = r.client.Get(ctx, client.ObjectKey{ + Name: test.pvb.Name, + Namespace: test.pvb.Namespace, + }, &pvb) + + if test.expectDeleted { + assert.True(t, apierrors.IsNotFound(err)) + } else { + require.NoError(t, err) + + assert.Equal(t, test.expected.Status.Phase, pvb.Status.Phase) + assert.Contains(t, pvb.Status.Message, test.expected.Status.Message) + assert.Equal(t, pvb.Finalizers, test.expected.Finalizers) + assert.Equal(t, pvb.Spec.Cancel, test.expected.Spec.Cancel) + } + } + + if !test.expectDataPath { + assert.Nil(t, r.dataPathMgr.GetAsyncBR(test.pvb.Name)) } else { - Expect(err).ToNot(HaveOccurred()) - Eventually(pvb.Status.Phase).Should(Equal(test.expected.Status.Phase)) + assert.NotNil(t, r.dataPathMgr.GetAsyncBR(test.pvb.Name)) } - // Processed PVBs will have completion timestamps. - if test.expectedProcessed == true { - Expect(pvb.Status.CompletionTimestamp).ToNot(BeNil()) + if test.expectCancelRecord { + assert.Contains(t, r.cancelledPVB, test.pvb.Name) + } else { + assert.Empty(t, r.cancelledPVB) } + }) + } +} - // Unprocessed PVBs will not have completion timestamps. - if test.expectedProcessed == false { - Expect(pvb.Status.CompletionTimestamp).To(BeNil()) - } +func TestOnPVBCancelled(t *testing.T) { + ctx := context.TODO() + r, err := initPVBReconciler() + require.NoError(t, err) + pvb := pvbBuilder().Result() + namespace := pvb.Namespace + pvbName := pvb.Name + + assert.NoError(t, r.client.Create(ctx, pvb)) + + r.OnDataPathCancelled(ctx, namespace, pvbName) + updatedPvb := &velerov1api.PodVolumeBackup{} + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) + assert.Equal(t, velerov1api.PodVolumeBackupPhaseCanceled, updatedPvb.Status.Phase) + assert.False(t, updatedPvb.Status.CompletionTimestamp.IsZero()) + assert.False(t, updatedPvb.Status.StartTimestamp.IsZero()) +} + +func TestOnPVBProgress(t *testing.T) { + totalBytes := int64(1024) + bytesDone := int64(512) + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + progress uploader.Progress + needErrs []bool + }{ + { + name: "patch in progress phase success", + pvb: pvbBuilder().Result(), + progress: uploader.Progress{ + TotalBytes: totalBytes, + BytesDone: bytesDone, + }, }, - Entry("empty phase pvb on same node should be processed", request{ - pvb: pvbBuilder().Phase("").Node("test_node").Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: true, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseCompleted). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("new phase pvb on same node should be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseNew). - Node("test_node"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: true, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseCompleted). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("in progress phase pvb on same node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseInProgress). - Node("test_node"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseInProgress). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("completed phase pvb on same node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseCompleted). - Node("test_node"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseCompleted). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("failed phase pvb on same node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseFailed). - Node("test_node"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseFailed). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("empty phase pvb on different node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseFailed). - Node("test_node_2"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseFailed). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("new phase pvb on different node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseNew). - Node("test_node_2"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseNew). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("in progress phase pvb on different node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseInProgress). - Node("test_node_2"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseInProgress). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("completed phase pvb on different node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseCompleted). - Node("test_node_2"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseCompleted). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("failed phase pvb on different node should not be processed", request{ - pvb: pvbBuilder(). - Phase(velerov1api.PodVolumeBackupPhaseFailed). - Node("test_node_2"). - Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(velerov1api.PodVolumeBackupPhaseFailed). - Result(), - expectedRequeue: ctrl.Result{}, - }), - Entry("pvb should be requeued when exceeding max concurrent number", request{ - pvb: pvbBuilder().Phase("").Node("test_node").Result(), - pod: podBuilder().Result(), - bsl: bslBuilder().Result(), - backupRepo: buildBackupRepo(), - dataMgr: datapath.NewManager(0), - expectedProcessed: false, - expected: builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1"). - Phase(""). - Result(), - expectedRequeue: ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, - }), - ) -}) + { + name: "failed to get pvb", + pvb: pvbBuilder().Result(), + needErrs: []bool{true, false, false, false}, + }, + { + name: "failed to patch pvb", + pvb: pvbBuilder().Result(), + needErrs: []bool{false, false, true, false}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.TODO() + + r, err := initPVBReconciler(test.needErrs...) + require.NoError(t, err) + defer func() { + r.client.Delete(ctx, test.pvb, &client.DeleteOptions{}) + }() + + pvb := pvbBuilder().Result() + namespace := pvb.Namespace + pvbName := pvb.Name + + assert.NoError(t, r.client.Create(context.Background(), pvb)) + + // Create a Progress object + progress := &uploader.Progress{ + TotalBytes: totalBytes, + BytesDone: bytesDone, + } + + r.OnDataPathProgress(ctx, namespace, pvbName, progress) + if len(test.needErrs) != 0 && !test.needErrs[0] { + updatedPvb := &velerov1api.PodVolumeBackup{} + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) + assert.Equal(t, test.progress.TotalBytes, updatedPvb.Status.Progress.TotalBytes) + assert.Equal(t, test.progress.BytesDone, updatedPvb.Status.Progress.BytesDone) + } + }) + } +} + +func TestOnPvbFailed(t *testing.T) { + ctx := context.TODO() + r, err := initPVBReconciler() + require.NoError(t, err) + + pvb := pvbBuilder().Result() + namespace := pvb.Namespace + pvbName := pvb.Name + + assert.NoError(t, r.client.Create(ctx, pvb)) + + r.OnDataPathFailed(ctx, namespace, pvbName, fmt.Errorf("Failed to handle %v", pvbName)) + updatedPvb := &velerov1api.PodVolumeBackup{} + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) + assert.Equal(t, velerov1api.PodVolumeBackupPhaseFailed, updatedPvb.Status.Phase) + assert.False(t, updatedPvb.Status.CompletionTimestamp.IsZero()) + assert.False(t, updatedPvb.Status.StartTimestamp.IsZero()) +} + +func TestOnPvbCompleted(t *testing.T) { + ctx := context.TODO() + r, err := initPVBReconciler() + require.NoError(t, err) + + now := time.Now() + pvb := pvbBuilder().StartTimestamp(&metav1.Time{Time: now.Add(-time.Minute)}).CompletionTimestamp(&metav1.Time{Time: now}).OwnerReference(metav1.OwnerReference{Name: "test-backup"}).Result() + namespace := pvb.Namespace + pvbName := pvb.Name + + assert.NoError(t, r.client.Create(ctx, pvb)) + + r.OnDataPathCompleted(ctx, namespace, pvbName, datapath.Result{}) + updatedPvb := &velerov1api.PodVolumeBackup{} + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) + assert.Equal(t, velerov1api.PodVolumeBackupPhaseCompleted, updatedPvb.Status.Phase) + assert.False(t, updatedPvb.Status.CompletionTimestamp.IsZero()) +} + +func TestFindPvbForPod(t *testing.T) { + r, err := initPVBReconciler() + require.NoError(t, err) + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + pod *corev1api.Pod + checkFunc func(*velerov1api.PodVolumeBackup, []reconcile.Request) + }{ + { + name: "find pvb for pod", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Labels(map[string]string{velerov1api.PVBLabel: pvbName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(), + checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { + // Assert that the function returns a single request + assert.Len(t, requests, 1) + // Assert that the request contains the correct namespaced name + assert.Equal(t, pvb.Namespace, requests[0].Namespace) + assert.Equal(t, pvb.Name, requests[0].Name) + }, + }, { + name: "no selected label found for pod", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Result(), + checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { + // Assert that the function returns a single request + assert.Empty(t, requests) + }, + }, { + name: "no matched pod", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseAccepted).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Labels(map[string]string{velerov1api.PVBLabel: "non-existing-pvb"}).Result(), + checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { + assert.Empty(t, requests) + }, + }, + { + name: "pvb not accepte", + pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvbName).Labels(map[string]string{velerov1api.PVBLabel: pvbName}).Result(), + checkFunc: func(pvb *velerov1api.PodVolumeBackup, requests []reconcile.Request) { + assert.Empty(t, requests) + }, + }, + } + for _, test := range tests { + ctx := context.Background() + assert.NoError(t, r.client.Create(ctx, test.pod)) + assert.NoError(t, r.client.Create(ctx, test.pvb)) + + requests := r.findPVBForPod(context.Background(), test.pod) + test.checkFunc(test.pvb, requests) + r.client.Delete(ctx, test.pvb, &client.DeleteOptions{}) + if test.pod != nil { + r.client.Delete(ctx, test.pod, &client.DeleteOptions{}) + } + } +} + +func TestAcceptPvb(t *testing.T) { + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + needErrs []error + expectedErr string + }{ + { + name: "update fail", + pvb: pvbBuilder().Node("test-node").Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + expectedErr: "error updating PVB with error velero/pvb-1: fake-update-error", + }, + { + name: "succeed", + pvb: pvbBuilder().Node("test-node").Result(), + needErrs: []error{nil, nil, nil, nil}, + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initPVBReconcilerWithError(test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.pvb) + require.NoError(t, err) + + err = r.acceptPodVolumeBackup(ctx, test.pvb) + if test.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, test.expectedErr) + } + } +} + +func TestOnPvbPrepareTimeout(t *testing.T) { + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + needErrs []error + expected *velerov1api.PodVolumeBackup + }{ + { + name: "update fail", + pvb: pvbBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + expected: pvbBuilder().Result(), + }, + { + name: "update interrupted", + pvb: pvbBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + expected: pvbBuilder().Result(), + }, + { + name: "succeed", + pvb: pvbBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhaseFailed).Result(), + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initPVBReconcilerWithError(test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.pvb) + require.NoError(t, err) + + r.onPrepareTimeout(ctx, test.pvb) + + pvb := velerov1api.PodVolumeBackup{} + _ = r.client.Get(ctx, client.ObjectKey{ + Name: test.pvb.Name, + Namespace: test.pvb.Namespace, + }, &pvb) + + assert.Equal(t, test.expected.Status.Phase, pvb.Status.Phase) + } +} + +func TestTryCancelPvb(t *testing.T) { + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + needErrs []error + succeeded bool + expectedErr string + }{ + { + name: "update fail", + pvb: pvbBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + }, + { + name: "cancel by others", + pvb: pvbBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + }, + { + name: "succeed", + pvb: pvbBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + succeeded: true, + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initPVBReconcilerWithError(test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.pvb) + require.NoError(t, err) + + r.tryCancelPodVolumeBackup(ctx, test.pvb, "") + + if test.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, test.expectedErr) + } + } +} + +func TestUpdatePvbWithRetry(t *testing.T) { + namespacedName := types.NamespacedName{ + Name: pvbName, + Namespace: "velero", + } + + // Define test cases + testCases := []struct { + Name string + needErrs []bool + noChange bool + ExpectErr bool + }{ + { + Name: "SuccessOnFirstAttempt", + }, + { + Name: "Error get", + needErrs: []bool{true, false, false, false, false}, + ExpectErr: true, + }, + { + Name: "Error update", + needErrs: []bool{false, false, true, false, false}, + ExpectErr: true, + }, + { + Name: "no change", + noChange: true, + needErrs: []bool{false, false, true, false, false}, + }, + { + Name: "Conflict with error timeout", + needErrs: []bool{false, false, false, false, true}, + ExpectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + ctx, cancelFunc := context.WithTimeout(context.TODO(), time.Second*5) + defer cancelFunc() + r, err := initPVBReconciler(tc.needErrs...) + require.NoError(t, err) + err = r.client.Create(ctx, pvbBuilder().Result()) + require.NoError(t, err) + updateFunc := func(pvb *velerov1api.PodVolumeBackup) bool { + if tc.noChange { + return false + } + + pvb.Spec.Cancel = true + return true + } + err = UpdatePVBWithRetry(ctx, r.client, namespacedName, velerotest.NewLogger().WithField("name", tc.Name), updateFunc) + if tc.ExpectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 776cdc332..6a96617f9 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -169,7 +169,8 @@ func newBackupper( } if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted && - pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseFailed { + pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseFailed && + pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCanceled { return } @@ -179,7 +180,8 @@ func newBackupper( existPVB, ok := existObj.(*velerov1api.PodVolumeBackup) // the PVB in the indexer is already in final status, no need to call WaitGroup.Done() if ok && (existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || - existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed) { + existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || + pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled) { statusChangedToFinal = false } } @@ -428,7 +430,7 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero continue } podVolumeBackups = append(podVolumeBackups, pvb) - if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { + if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { log.Errorf("pod volume backup failed: %s", pvb.Status.Message) } }